{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_steps:  1600\n",
      "device:  cuda:0\n",
      "state dim:  24\n",
      "action dim:  Box(4,)\n",
      "leraning rate:  8e-05\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import torch\n",
    "from sac_agent import soft_actor_critic_agent\n",
    "from replay_memory import ReplayMemory\n",
    "\n",
    "seed=0  \n",
    "env = gym.make('BipedalWalker-v2')\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "env.seed(seed)\n",
    "max_steps = env._max_episode_steps\n",
    "print('max_steps: ', max_steps)\n",
    "\n",
    "batch_size=256\n",
    "\n",
    "LEARNING_RATE=0.00008 # lr = 0.0001 for BipedalWalker-SAC_lr0001\n",
    "eval=True  ## \n",
    "start_steps=10000 ## Steps sampling random actions\n",
    "replay_size=1000000 ## size of replay buffer\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# alpha=0.2  # relative importance of the entropy\n",
    "# gamma=0.99  # discount factor \n",
    "# tau=0.005  # target smoothing coefficient(τ)\n",
    "\n",
    "agent = soft_actor_critic_agent(env.observation_space.shape[0], env.action_space, \\\n",
    "        device=device, hidden_size=256, lr=LEARNING_RATE, gamma=0.99, tau=0.005, alpha=0.2)\n",
    "\n",
    "memory = ReplayMemory(replay_size)\n",
    "\n",
    "print('device: ', device)\n",
    "print('state dim: ', env.observation_space.shape[0])\n",
    "print('action dim: ', env.action_space)\n",
    "print('leraning rate: ', LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(agent, directory, filename, suffix):\n",
    "    torch.save(agent.policy.state_dict(), '%s/%s_actor_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(agent.critic.state_dict(), '%s/%s_critic_%s.pth' % (directory, filename, suffix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 0, Total Steps: 77, Ep.Steps: 77, Score: -114.65, Avg.Score: -114.65, Time: 00:00:00\n",
      "Ep.: 1, Total Steps: 1677, Ep.Steps: 1600, Score: -78.28, Avg.Score: -96.46, Time: 00:01:32\n",
      "Ep.: 2, Total Steps: 1741, Ep.Steps: 64, Score: -111.78, Avg.Score: -101.57, Time: 00:01:36\n",
      "Ep.: 3, Total Steps: 3341, Ep.Steps: 1600, Score: -86.86, Avg.Score: -97.89, Time: 00:03:11\n",
      "Ep.: 4, Total Steps: 3388, Ep.Steps: 47, Score: -109.59, Avg.Score: -100.23, Time: 00:03:13\n",
      "Ep.: 5, Total Steps: 4988, Ep.Steps: 1600, Score: -88.05, Avg.Score: -98.20, Time: 00:04:47\n",
      "Ep.: 6, Total Steps: 6588, Ep.Steps: 1600, Score: -80.07, Avg.Score: -95.61, Time: 00:06:23\n",
      "Ep.: 7, Total Steps: 8188, Ep.Steps: 1600, Score: -91.96, Avg.Score: -95.15, Time: 00:08:02\n",
      "Ep.: 8, Total Steps: 8260, Ep.Steps: 72, Score: -106.64, Avg.Score: -96.43, Time: 00:08:07\n",
      "Ep.: 9, Total Steps: 8383, Ep.Steps: 123, Score: -98.78, Avg.Score: -96.67, Time: 00:08:15\n",
      "Ep.: 10, Total Steps: 9983, Ep.Steps: 1600, Score: -86.57, Avg.Score: -95.75, Time: 00:10:00\n",
      "Ep.: 11, Total Steps: 10051, Ep.Steps: 68, Score: -104.01, Avg.Score: -96.44, Time: 00:10:05\n",
      "Ep.: 12, Total Steps: 10112, Ep.Steps: 61, Score: -101.65, Avg.Score: -96.84, Time: 00:10:09\n",
      "Ep.: 13, Total Steps: 10172, Ep.Steps: 60, Score: -109.78, Avg.Score: -97.76, Time: 00:10:13\n",
      "Ep.: 14, Total Steps: 10238, Ep.Steps: 66, Score: -108.60, Avg.Score: -98.48, Time: 00:10:17\n",
      "Ep.: 15, Total Steps: 10313, Ep.Steps: 75, Score: -102.44, Avg.Score: -98.73, Time: 00:10:23\n",
      "Ep.: 16, Total Steps: 10370, Ep.Steps: 57, Score: -101.75, Avg.Score: -98.91, Time: 00:10:26\n",
      "Ep.: 17, Total Steps: 10428, Ep.Steps: 58, Score: -107.31, Avg.Score: -99.38, Time: 00:10:30\n",
      "Ep.: 18, Total Steps: 10492, Ep.Steps: 64, Score: -102.33, Avg.Score: -99.53, Time: 00:10:34\n",
      "Ep.: 19, Total Steps: 10581, Ep.Steps: 89, Score: -105.02, Avg.Score: -99.81, Time: 00:10:40\n",
      "Ep.: 20, Total Steps: 10653, Ep.Steps: 72, Score: -101.04, Avg.Score: -99.86, Time: 00:10:45\n",
      "Ep.: 21, Total Steps: 12253, Ep.Steps: 1600, Score: -95.54, Avg.Score: -99.67, Time: 00:12:30\n",
      "Ep.: 22, Total Steps: 12362, Ep.Steps: 109, Score: -107.38, Avg.Score: -100.00, Time: 00:12:37\n",
      "Ep.: 23, Total Steps: 13962, Ep.Steps: 1600, Score: -85.15, Avg.Score: -99.38, Time: 00:14:23\n",
      "Ep.: 24, Total Steps: 14027, Ep.Steps: 65, Score: -115.54, Avg.Score: -100.03, Time: 00:14:28\n",
      "Ep.: 25, Total Steps: 15627, Ep.Steps: 1600, Score: -77.14, Avg.Score: -99.15, Time: 00:16:17\n",
      "Ep.: 26, Total Steps: 15684, Ep.Steps: 57, Score: -108.14, Avg.Score: -99.48, Time: 00:16:21\n",
      "Ep.: 27, Total Steps: 16614, Ep.Steps: 930, Score: -127.40, Avg.Score: -100.48, Time: 00:17:24\n",
      "Ep.: 28, Total Steps: 16680, Ep.Steps: 66, Score: -122.30, Avg.Score: -101.23, Time: 00:17:29\n",
      "Ep.: 29, Total Steps: 16759, Ep.Steps: 79, Score: -104.53, Avg.Score: -101.34, Time: 00:17:35\n",
      "Ep.: 30, Total Steps: 16866, Ep.Steps: 107, Score: -120.80, Avg.Score: -101.97, Time: 00:17:43\n",
      "Ep.: 31, Total Steps: 16973, Ep.Steps: 107, Score: -113.22, Avg.Score: -102.32, Time: 00:17:51\n",
      "Ep.: 32, Total Steps: 18573, Ep.Steps: 1600, Score: -89.44, Avg.Score: -101.93, Time: 00:19:39\n",
      "Ep.: 33, Total Steps: 20173, Ep.Steps: 1600, Score: -116.01, Avg.Score: -102.35, Time: 00:21:17\n",
      "Ep.: 34, Total Steps: 21773, Ep.Steps: 1600, Score: -104.20, Avg.Score: -102.40, Time: 00:22:54\n",
      "Ep.: 35, Total Steps: 23373, Ep.Steps: 1600, Score: -63.20, Avg.Score: -101.31, Time: 00:24:31\n",
      "Ep.: 36, Total Steps: 23451, Ep.Steps: 78, Score: -111.79, Avg.Score: -101.59, Time: 00:24:36\n",
      "Ep.: 37, Total Steps: 23585, Ep.Steps: 134, Score: -109.78, Avg.Score: -101.81, Time: 00:24:44\n",
      "Ep.: 38, Total Steps: 23629, Ep.Steps: 44, Score: -110.81, Avg.Score: -102.04, Time: 00:24:47\n",
      "Ep.: 39, Total Steps: 23693, Ep.Steps: 64, Score: -113.85, Avg.Score: -102.33, Time: 00:24:53\n",
      "Ep.: 40, Total Steps: 23753, Ep.Steps: 60, Score: -112.75, Avg.Score: -102.59, Time: 00:24:58\n",
      "Ep.: 41, Total Steps: 23830, Ep.Steps: 77, Score: -116.98, Avg.Score: -102.93, Time: 00:25:03\n",
      "Ep.: 42, Total Steps: 25430, Ep.Steps: 1600, Score: -84.21, Avg.Score: -102.50, Time: 00:26:56\n",
      "Ep.: 43, Total Steps: 25556, Ep.Steps: 126, Score: -114.63, Avg.Score: -102.77, Time: 00:27:05\n",
      "Ep.: 44, Total Steps: 27156, Ep.Steps: 1600, Score: -98.38, Avg.Score: -102.67, Time: 00:28:50\n",
      "Ep.: 45, Total Steps: 28756, Ep.Steps: 1600, Score: -82.18, Avg.Score: -102.23, Time: 00:30:27\n",
      "Ep.: 46, Total Steps: 30356, Ep.Steps: 1600, Score: -71.04, Avg.Score: -101.56, Time: 00:32:10\n",
      "Ep.: 47, Total Steps: 31956, Ep.Steps: 1600, Score: -93.98, Avg.Score: -101.41, Time: 00:33:52\n",
      "Ep.: 48, Total Steps: 32038, Ep.Steps: 82, Score: -105.10, Avg.Score: -101.48, Time: 00:33:57\n",
      "Ep.: 49, Total Steps: 32111, Ep.Steps: 73, Score: -104.10, Avg.Score: -101.53, Time: 00:34:02\n",
      "Ep.: 50, Total Steps: 33711, Ep.Steps: 1600, Score: -82.39, Avg.Score: -101.16, Time: 00:35:44\n",
      "Ep.: 51, Total Steps: 35311, Ep.Steps: 1600, Score: -72.18, Avg.Score: -100.60, Time: 00:37:20\n",
      "Ep.: 52, Total Steps: 36911, Ep.Steps: 1600, Score: -73.46, Avg.Score: -100.09, Time: 00:38:55\n",
      "Ep.: 53, Total Steps: 38511, Ep.Steps: 1600, Score: -89.18, Avg.Score: -99.89, Time: 00:40:30\n",
      "Ep.: 54, Total Steps: 40111, Ep.Steps: 1600, Score: -149.65, Avg.Score: -100.79, Time: 00:42:03\n",
      "Ep.: 55, Total Steps: 40215, Ep.Steps: 104, Score: -113.04, Avg.Score: -101.01, Time: 00:42:09\n",
      "Ep.: 56, Total Steps: 40390, Ep.Steps: 175, Score: -120.72, Avg.Score: -101.36, Time: 00:42:19\n",
      "Ep.: 57, Total Steps: 40459, Ep.Steps: 69, Score: -114.69, Avg.Score: -101.59, Time: 00:42:23\n",
      "Ep.: 58, Total Steps: 40538, Ep.Steps: 79, Score: -128.42, Avg.Score: -102.04, Time: 00:42:28\n",
      "Ep.: 59, Total Steps: 40619, Ep.Steps: 81, Score: -114.89, Avg.Score: -102.26, Time: 00:42:33\n",
      "Ep.: 60, Total Steps: 40679, Ep.Steps: 60, Score: -107.98, Avg.Score: -102.35, Time: 00:42:36\n",
      "Ep.: 61, Total Steps: 40784, Ep.Steps: 105, Score: -111.72, Avg.Score: -102.50, Time: 00:42:43\n",
      "Ep.: 62, Total Steps: 42384, Ep.Steps: 1600, Score: -101.67, Avg.Score: -102.49, Time: 00:44:16\n",
      "Ep.: 63, Total Steps: 43984, Ep.Steps: 1600, Score: -82.17, Avg.Score: -102.17, Time: 00:45:50\n",
      "Ep.: 64, Total Steps: 45584, Ep.Steps: 1600, Score: -79.16, Avg.Score: -101.82, Time: 00:47:33\n",
      "Ep.: 65, Total Steps: 47184, Ep.Steps: 1600, Score: -71.87, Avg.Score: -101.36, Time: 00:49:12\n",
      "Ep.: 66, Total Steps: 48784, Ep.Steps: 1600, Score: -71.91, Avg.Score: -100.92, Time: 00:50:49\n",
      "Ep.: 67, Total Steps: 50384, Ep.Steps: 1600, Score: -66.83, Avg.Score: -100.42, Time: 00:52:25\n",
      "Ep.: 68, Total Steps: 51984, Ep.Steps: 1600, Score: -80.20, Avg.Score: -100.13, Time: 00:54:04\n",
      "Ep.: 69, Total Steps: 53584, Ep.Steps: 1600, Score: -89.05, Avg.Score: -99.97, Time: 00:55:41\n",
      "Ep.: 70, Total Steps: 55184, Ep.Steps: 1600, Score: -66.17, Avg.Score: -99.49, Time: 00:57:17\n",
      "Ep.: 71, Total Steps: 56784, Ep.Steps: 1600, Score: -20.02, Avg.Score: -98.39, Time: 00:58:54\n",
      "Ep.: 72, Total Steps: 58384, Ep.Steps: 1600, Score: 20.54, Avg.Score: -96.76, Time: 01:00:30\n",
      "Ep.: 73, Total Steps: 59984, Ep.Steps: 1600, Score: -43.66, Avg.Score: -96.04, Time: 01:02:07\n",
      "Ep.: 74, Total Steps: 61584, Ep.Steps: 1600, Score: -78.04, Avg.Score: -95.80, Time: 01:03:44\n",
      "Ep.: 75, Total Steps: 63184, Ep.Steps: 1600, Score: -98.38, Avg.Score: -95.84, Time: 01:05:20\n",
      "Ep.: 76, Total Steps: 64784, Ep.Steps: 1600, Score: -114.62, Avg.Score: -96.08, Time: 01:06:57\n",
      "Ep.: 77, Total Steps: 66384, Ep.Steps: 1600, Score: -114.61, Avg.Score: -96.32, Time: 01:08:33\n",
      "Ep.: 78, Total Steps: 67984, Ep.Steps: 1600, Score: -102.29, Avg.Score: -96.39, Time: 01:10:10\n",
      "Ep.: 79, Total Steps: 69584, Ep.Steps: 1600, Score: -94.95, Avg.Score: -96.38, Time: 01:11:46\n",
      "Ep.: 80, Total Steps: 71184, Ep.Steps: 1600, Score: -70.38, Avg.Score: -96.05, Time: 01:13:23\n",
      "Ep.: 81, Total Steps: 72784, Ep.Steps: 1600, Score: -61.38, Avg.Score: -95.63, Time: 01:14:59\n",
      "Ep.: 82, Total Steps: 72851, Ep.Steps: 67, Score: -119.12, Avg.Score: -95.92, Time: 01:15:03\n",
      "Ep.: 83, Total Steps: 72926, Ep.Steps: 75, Score: -118.24, Avg.Score: -96.18, Time: 01:15:07\n",
      "Ep.: 84, Total Steps: 72996, Ep.Steps: 70, Score: -120.29, Avg.Score: -96.46, Time: 01:15:12\n",
      "Ep.: 85, Total Steps: 73052, Ep.Steps: 56, Score: -115.34, Avg.Score: -96.68, Time: 01:15:15\n",
      "Ep.: 86, Total Steps: 73153, Ep.Steps: 101, Score: -130.88, Avg.Score: -97.08, Time: 01:15:21\n",
      "Ep.: 87, Total Steps: 73234, Ep.Steps: 81, Score: -130.16, Avg.Score: -97.45, Time: 01:15:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 88, Total Steps: 73330, Ep.Steps: 96, Score: -130.62, Avg.Score: -97.83, Time: 01:15:32\n",
      "Ep.: 89, Total Steps: 73422, Ep.Steps: 92, Score: -131.31, Avg.Score: -98.20, Time: 01:15:38\n",
      "Ep.: 90, Total Steps: 73502, Ep.Steps: 80, Score: -120.89, Avg.Score: -98.45, Time: 01:15:42\n",
      "Ep.: 91, Total Steps: 73599, Ep.Steps: 97, Score: -131.24, Avg.Score: -98.80, Time: 01:15:48\n",
      "Ep.: 92, Total Steps: 73677, Ep.Steps: 78, Score: -129.27, Avg.Score: -99.13, Time: 01:15:53\n",
      "Ep.: 93, Total Steps: 73767, Ep.Steps: 90, Score: -133.08, Avg.Score: -99.49, Time: 01:15:58\n",
      "Ep.: 94, Total Steps: 73844, Ep.Steps: 77, Score: -125.95, Avg.Score: -99.77, Time: 01:16:03\n",
      "Ep.: 95, Total Steps: 73924, Ep.Steps: 80, Score: -126.26, Avg.Score: -100.05, Time: 01:16:08\n",
      "Ep.: 96, Total Steps: 74159, Ep.Steps: 235, Score: -140.07, Avg.Score: -100.46, Time: 01:16:22\n",
      "Ep.: 97, Total Steps: 74238, Ep.Steps: 79, Score: -129.29, Avg.Score: -100.75, Time: 01:16:27\n",
      "Ep.: 98, Total Steps: 74325, Ep.Steps: 87, Score: -129.05, Avg.Score: -101.04, Time: 01:16:32\n",
      "Ep.: 99, Total Steps: 74422, Ep.Steps: 97, Score: -132.97, Avg.Score: -101.36, Time: 01:16:38\n",
      "Ep.: 100, Total Steps: 76022, Ep.Steps: 1600, Score: -49.92, Avg.Score: -100.71, Time: 01:18:15\n",
      "Ep.: 101, Total Steps: 76201, Ep.Steps: 179, Score: -118.88, Avg.Score: -101.12, Time: 01:18:26\n",
      "Ep.: 102, Total Steps: 77801, Ep.Steps: 1600, Score: 1.87, Avg.Score: -99.98, Time: 01:20:02\n",
      "Ep.: 103, Total Steps: 77885, Ep.Steps: 84, Score: -117.52, Avg.Score: -100.29, Time: 01:20:07\n",
      "Ep.: 104, Total Steps: 77957, Ep.Steps: 72, Score: -117.20, Avg.Score: -100.36, Time: 01:20:12\n",
      "Ep.: 105, Total Steps: 79546, Ep.Steps: 1589, Score: -95.66, Avg.Score: -100.44, Time: 01:21:47\n",
      "Ep.: 106, Total Steps: 81146, Ep.Steps: 1600, Score: 12.09, Avg.Score: -99.52, Time: 01:23:24\n",
      "Ep.: 107, Total Steps: 82746, Ep.Steps: 1600, Score: -24.20, Avg.Score: -98.84, Time: 01:25:00\n",
      "Ep.: 108, Total Steps: 84346, Ep.Steps: 1600, Score: -132.37, Avg.Score: -99.10, Time: 01:26:37\n",
      "Ep.: 109, Total Steps: 85946, Ep.Steps: 1600, Score: -26.03, Avg.Score: -98.37, Time: 01:28:13\n",
      "Ep.: 110, Total Steps: 87546, Ep.Steps: 1600, Score: -134.60, Avg.Score: -98.85, Time: 01:29:49\n",
      "Ep.: 111, Total Steps: 89146, Ep.Steps: 1600, Score: -61.64, Avg.Score: -98.43, Time: 01:31:26\n",
      "Ep.: 112, Total Steps: 90746, Ep.Steps: 1600, Score: -80.18, Avg.Score: -98.21, Time: 01:33:11\n",
      "Ep.: 113, Total Steps: 92346, Ep.Steps: 1600, Score: -84.68, Avg.Score: -97.96, Time: 01:34:50\n",
      "Ep.: 114, Total Steps: 93946, Ep.Steps: 1600, Score: -85.14, Avg.Score: -97.73, Time: 01:36:27\n",
      "Ep.: 115, Total Steps: 95546, Ep.Steps: 1600, Score: -66.81, Avg.Score: -97.37, Time: 01:38:03\n",
      "Ep.: 116, Total Steps: 97146, Ep.Steps: 1600, Score: -71.13, Avg.Score: -97.06, Time: 01:39:40\n",
      "Ep.: 117, Total Steps: 98746, Ep.Steps: 1600, Score: -61.67, Avg.Score: -96.61, Time: 01:41:16\n",
      "Ep.: 118, Total Steps: 100346, Ep.Steps: 1600, Score: -44.29, Avg.Score: -96.03, Time: 01:42:52\n",
      "Ep.: 119, Total Steps: 101946, Ep.Steps: 1600, Score: -46.58, Avg.Score: -95.44, Time: 01:44:29\n",
      "Ep.: 120, Total Steps: 103546, Ep.Steps: 1600, Score: -47.59, Avg.Score: -94.91, Time: 01:46:05\n",
      "Ep.: 121, Total Steps: 105146, Ep.Steps: 1600, Score: -13.54, Avg.Score: -94.09, Time: 01:47:41\n",
      "Ep.: 122, Total Steps: 106746, Ep.Steps: 1600, Score: -14.19, Avg.Score: -93.16, Time: 01:49:17\n",
      "Ep.: 123, Total Steps: 108346, Ep.Steps: 1600, Score: 14.74, Avg.Score: -92.16, Time: 01:50:54\n",
      "Ep.: 124, Total Steps: 109946, Ep.Steps: 1600, Score: -1.02, Avg.Score: -91.01, Time: 01:52:30\n",
      "Ep.: 125, Total Steps: 111546, Ep.Steps: 1600, Score: -7.30, Avg.Score: -90.31, Time: 01:54:07\n",
      "Ep.: 126, Total Steps: 113146, Ep.Steps: 1600, Score: -3.31, Avg.Score: -89.27, Time: 01:55:43\n",
      "Ep.: 127, Total Steps: 114746, Ep.Steps: 1600, Score: 7.00, Avg.Score: -87.92, Time: 01:57:19\n",
      "Ep.: 128, Total Steps: 116346, Ep.Steps: 1600, Score: 0.04, Avg.Score: -86.70, Time: 01:58:56\n",
      "Ep.: 129, Total Steps: 117946, Ep.Steps: 1600, Score: -7.23, Avg.Score: -85.72, Time: 02:00:32\n",
      "Ep.: 130, Total Steps: 119546, Ep.Steps: 1600, Score: -30.53, Avg.Score: -84.82, Time: 02:02:08\n",
      "Ep.: 131, Total Steps: 121146, Ep.Steps: 1600, Score: -37.37, Avg.Score: -84.06, Time: 02:03:45\n",
      "Ep.: 132, Total Steps: 122746, Ep.Steps: 1600, Score: -34.22, Avg.Score: -83.51, Time: 02:05:21\n",
      "Ep.: 133, Total Steps: 122886, Ep.Steps: 140, Score: -88.29, Avg.Score: -83.23, Time: 02:05:29\n",
      "Ep.: 134, Total Steps: 123160, Ep.Steps: 274, Score: -91.43, Avg.Score: -83.11, Time: 02:05:46\n",
      "Ep.: 135, Total Steps: 123266, Ep.Steps: 106, Score: -101.38, Avg.Score: -83.49, Time: 02:05:52\n",
      "Ep.: 136, Total Steps: 123370, Ep.Steps: 104, Score: -95.06, Avg.Score: -83.32, Time: 02:05:58\n",
      "Ep.: 137, Total Steps: 123427, Ep.Steps: 57, Score: -105.92, Avg.Score: -83.28, Time: 02:06:02\n",
      "Ep.: 138, Total Steps: 124786, Ep.Steps: 1359, Score: -54.84, Avg.Score: -82.72, Time: 02:07:24\n",
      "Ep.: 139, Total Steps: 125762, Ep.Steps: 976, Score: -77.13, Avg.Score: -82.36, Time: 02:08:23\n",
      "Ep.: 140, Total Steps: 125927, Ep.Steps: 165, Score: -84.84, Avg.Score: -82.08, Time: 02:08:33\n",
      "Ep.: 141, Total Steps: 127527, Ep.Steps: 1600, Score: 12.53, Avg.Score: -80.78, Time: 02:10:09\n",
      "Ep.: 142, Total Steps: 129127, Ep.Steps: 1600, Score: -40.22, Avg.Score: -80.34, Time: 02:11:45\n",
      "Ep.: 143, Total Steps: 130727, Ep.Steps: 1600, Score: 11.58, Avg.Score: -79.08, Time: 02:13:22\n",
      "Ep.: 144, Total Steps: 132327, Ep.Steps: 1600, Score: -46.70, Avg.Score: -78.56, Time: 02:14:58\n",
      "Ep.: 145, Total Steps: 132747, Ep.Steps: 420, Score: -152.36, Avg.Score: -79.26, Time: 02:15:23\n",
      "Ep.: 146, Total Steps: 134347, Ep.Steps: 1600, Score: 19.57, Avg.Score: -78.36, Time: 02:17:01\n",
      "Ep.: 147, Total Steps: 135947, Ep.Steps: 1600, Score: -50.95, Avg.Score: -77.93, Time: 02:18:39\n",
      "Ep.: 148, Total Steps: 137547, Ep.Steps: 1600, Score: 18.79, Avg.Score: -76.69, Time: 02:20:16\n",
      "Ep.: 149, Total Steps: 139147, Ep.Steps: 1600, Score: 7.17, Avg.Score: -75.58, Time: 02:21:53\n",
      "Ep.: 150, Total Steps: 140747, Ep.Steps: 1600, Score: 20.13, Avg.Score: -74.55, Time: 02:23:29\n",
      "Ep.: 151, Total Steps: 142347, Ep.Steps: 1600, Score: 23.83, Avg.Score: -73.59, Time: 02:25:06\n",
      "Ep.: 152, Total Steps: 143947, Ep.Steps: 1600, Score: 35.96, Avg.Score: -72.50, Time: 02:26:43\n",
      "Ep.: 153, Total Steps: 145547, Ep.Steps: 1600, Score: 39.69, Avg.Score: -71.21, Time: 02:28:20\n",
      "Ep.: 154, Total Steps: 147147, Ep.Steps: 1600, Score: 54.89, Avg.Score: -69.16, Time: 02:29:56\n",
      "Ep.: 155, Total Steps: 148747, Ep.Steps: 1600, Score: 29.68, Avg.Score: -67.74, Time: 02:31:32\n",
      "Ep.: 156, Total Steps: 150293, Ep.Steps: 1546, Score: -84.58, Avg.Score: -67.37, Time: 02:33:05\n",
      "Ep.: 157, Total Steps: 151893, Ep.Steps: 1600, Score: 71.67, Avg.Score: -65.51, Time: 02:34:42\n",
      "Ep.: 158, Total Steps: 153493, Ep.Steps: 1600, Score: 66.31, Avg.Score: -63.56, Time: 02:36:18\n",
      "Ep.: 159, Total Steps: 155093, Ep.Steps: 1600, Score: 99.90, Avg.Score: -61.42, Time: 02:37:54\n",
      "Ep.: 160, Total Steps: 156693, Ep.Steps: 1600, Score: -93.37, Avg.Score: -61.27, Time: 02:39:31\n",
      "Ep.: 161, Total Steps: 158293, Ep.Steps: 1600, Score: -109.61, Avg.Score: -61.25, Time: 02:41:08\n",
      "Ep.: 162, Total Steps: 159893, Ep.Steps: 1600, Score: -105.54, Avg.Score: -61.29, Time: 02:42:44\n",
      "Ep.: 163, Total Steps: 161493, Ep.Steps: 1600, Score: -79.33, Avg.Score: -61.26, Time: 02:44:20\n",
      "Ep.: 164, Total Steps: 163093, Ep.Steps: 1600, Score: -1.53, Avg.Score: -60.48, Time: 02:45:57\n",
      "Ep.: 165, Total Steps: 163265, Ep.Steps: 172, Score: -101.46, Avg.Score: -60.78, Time: 02:46:07\n",
      "Ep.: 166, Total Steps: 163491, Ep.Steps: 226, Score: -89.28, Avg.Score: -60.95, Time: 02:46:20\n",
      "Ep.: 167, Total Steps: 165091, Ep.Steps: 1600, Score: -9.62, Avg.Score: -60.38, Time: 02:47:57\n",
      "Ep.: 168, Total Steps: 166691, Ep.Steps: 1600, Score: -50.02, Avg.Score: -60.08, Time: 02:49:33\n",
      "Ep.: 169, Total Steps: 168291, Ep.Steps: 1600, Score: -6.52, Avg.Score: -59.25, Time: 02:51:10\n",
      "Ep.: 170, Total Steps: 169891, Ep.Steps: 1600, Score: -28.33, Avg.Score: -58.87, Time: 02:52:46\n",
      "Ep.: 171, Total Steps: 171491, Ep.Steps: 1600, Score: -24.04, Avg.Score: -58.91, Time: 02:54:23\n",
      "Ep.: 172, Total Steps: 173091, Ep.Steps: 1600, Score: -50.65, Avg.Score: -59.63, Time: 02:55:59\n",
      "Ep.: 173, Total Steps: 174691, Ep.Steps: 1600, Score: -48.91, Avg.Score: -59.68, Time: 02:57:36\n",
      "Ep.: 174, Total Steps: 176291, Ep.Steps: 1600, Score: -60.69, Avg.Score: -59.51, Time: 02:59:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 175, Total Steps: 177891, Ep.Steps: 1600, Score: -59.22, Avg.Score: -59.11, Time: 03:00:49\n",
      "Ep.: 176, Total Steps: 179491, Ep.Steps: 1600, Score: -47.77, Avg.Score: -58.45, Time: 03:02:25\n",
      "Ep.: 177, Total Steps: 179954, Ep.Steps: 463, Score: -102.71, Avg.Score: -58.33, Time: 03:02:53\n",
      "Ep.: 178, Total Steps: 181554, Ep.Steps: 1600, Score: -16.08, Avg.Score: -57.46, Time: 03:04:34\n",
      "Ep.: 179, Total Steps: 182021, Ep.Steps: 467, Score: -103.20, Avg.Score: -57.55, Time: 03:05:02\n",
      "Ep.: 180, Total Steps: 182538, Ep.Steps: 517, Score: -57.25, Avg.Score: -57.42, Time: 03:05:33\n",
      "Ep.: 181, Total Steps: 182929, Ep.Steps: 391, Score: -90.93, Avg.Score: -57.71, Time: 03:05:57\n",
      "Ep.: 182, Total Steps: 184529, Ep.Steps: 1600, Score: 84.07, Avg.Score: -55.68, Time: 03:07:33\n",
      "Ep.: 183, Total Steps: 186129, Ep.Steps: 1600, Score: 179.26, Avg.Score: -52.70, Time: 03:09:10\n",
      "Ep.: 184, Total Steps: 187729, Ep.Steps: 1600, Score: 162.03, Avg.Score: -49.88, Time: 03:10:46\n",
      "Ep.: 185, Total Steps: 189321, Ep.Steps: 1592, Score: 59.47, Avg.Score: -48.13, Time: 03:12:22\n",
      "Ep.: 186, Total Steps: 190921, Ep.Steps: 1600, Score: 165.92, Avg.Score: -45.17, Time: 03:13:58\n",
      "Ep.: 187, Total Steps: 192521, Ep.Steps: 1600, Score: 207.42, Avg.Score: -41.79, Time: 03:15:35\n",
      "Ep.: 188, Total Steps: 194121, Ep.Steps: 1600, Score: 191.64, Avg.Score: -38.57, Time: 03:17:11\n",
      "Ep.: 189, Total Steps: 195721, Ep.Steps: 1600, Score: 177.16, Avg.Score: -35.48, Time: 03:18:47\n",
      "Ep.: 190, Total Steps: 197321, Ep.Steps: 1600, Score: 162.33, Avg.Score: -32.65, Time: 03:20:24\n",
      "Ep.: 191, Total Steps: 198921, Ep.Steps: 1600, Score: 183.24, Avg.Score: -29.51, Time: 03:22:00\n",
      "Ep.: 192, Total Steps: 200521, Ep.Steps: 1600, Score: 106.25, Avg.Score: -27.15, Time: 03:23:37\n",
      "Ep.: 193, Total Steps: 202121, Ep.Steps: 1600, Score: 157.49, Avg.Score: -24.24, Time: 03:25:14\n",
      "Ep.: 194, Total Steps: 203721, Ep.Steps: 1600, Score: 141.41, Avg.Score: -21.57, Time: 03:26:51\n",
      "Ep.: 195, Total Steps: 205321, Ep.Steps: 1600, Score: 177.05, Avg.Score: -18.54, Time: 03:28:28\n",
      "Ep.: 196, Total Steps: 206921, Ep.Steps: 1600, Score: 174.87, Avg.Score: -15.39, Time: 03:30:04\n",
      "Ep.: 197, Total Steps: 208521, Ep.Steps: 1600, Score: 190.01, Avg.Score: -12.20, Time: 03:31:41\n",
      "Ep.: 198, Total Steps: 210121, Ep.Steps: 1600, Score: 170.90, Avg.Score: -9.20, Time: 03:33:17\n",
      "Ep.: 199, Total Steps: 211721, Ep.Steps: 1600, Score: 177.96, Avg.Score: -6.09, Time: 03:34:53\n",
      "Ep.: 200, Total Steps: 213321, Ep.Steps: 1600, Score: 193.19, Avg.Score: -3.66, Time: 03:36:30\n",
      "Ep.: 201, Total Steps: 214921, Ep.Steps: 1600, Score: 204.32, Avg.Score: -0.42, Time: 03:38:06\n",
      "Ep.: 202, Total Steps: 216521, Ep.Steps: 1600, Score: 164.39, Avg.Score: 1.20, Time: 03:39:43\n",
      "Ep.: 203, Total Steps: 217429, Ep.Steps: 908, Score: 2.61, Avg.Score: 2.40, Time: 03:40:38\n",
      "Ep.: 204, Total Steps: 218891, Ep.Steps: 1462, Score: 51.21, Avg.Score: 4.09, Time: 03:42:06\n",
      "Ep.: 205, Total Steps: 220484, Ep.Steps: 1593, Score: 267.40, Avg.Score: 7.72, Time: 03:43:49\n",
      "Ep.: 206, Total Steps: 222084, Ep.Steps: 1600, Score: 259.14, Avg.Score: 10.19, Time: 03:45:28\n",
      "Ep.: 207, Total Steps: 223684, Ep.Steps: 1600, Score: 255.32, Avg.Score: 12.98, Time: 03:47:04\n",
      "Ep.: 208, Total Steps: 225141, Ep.Steps: 1457, Score: 274.11, Avg.Score: 17.05, Time: 03:48:32\n",
      "Ep.: 209, Total Steps: 226627, Ep.Steps: 1486, Score: 269.63, Avg.Score: 20.01, Time: 03:50:02\n",
      "Ep.: 210, Total Steps: 228075, Ep.Steps: 1448, Score: 267.46, Avg.Score: 24.03, Time: 03:51:29\n",
      "Ep.: 211, Total Steps: 229486, Ep.Steps: 1411, Score: 267.88, Avg.Score: 27.32, Time: 03:52:54\n",
      "Ep.: 212, Total Steps: 230965, Ep.Steps: 1479, Score: 266.84, Avg.Score: 30.79, Time: 03:54:23\n",
      "Ep.: 213, Total Steps: 232418, Ep.Steps: 1453, Score: 268.79, Avg.Score: 34.33, Time: 03:55:51\n",
      "Ep.: 214, Total Steps: 233625, Ep.Steps: 1207, Score: 77.18, Avg.Score: 35.95, Time: 03:57:04\n",
      "Ep.: 215, Total Steps: 234072, Ep.Steps: 447, Score: -50.50, Avg.Score: 36.11, Time: 03:57:30\n",
      "Ep.: 216, Total Steps: 235595, Ep.Steps: 1523, Score: 262.70, Avg.Score: 39.45, Time: 03:59:02\n",
      "Ep.: 217, Total Steps: 237127, Ep.Steps: 1532, Score: 258.08, Avg.Score: 42.65, Time: 04:00:35\n",
      "Ep.: 218, Total Steps: 238727, Ep.Steps: 1600, Score: 249.94, Avg.Score: 45.59, Time: 04:02:11\n",
      "Ep.: 219, Total Steps: 240327, Ep.Steps: 1600, Score: 244.36, Avg.Score: 48.50, Time: 04:03:48\n",
      "Ep.: 220, Total Steps: 241907, Ep.Steps: 1580, Score: 254.81, Avg.Score: 51.52, Time: 04:05:27\n",
      "Ep.: 221, Total Steps: 243462, Ep.Steps: 1555, Score: 260.63, Avg.Score: 54.27, Time: 04:07:06\n",
      "Ep.: 222, Total Steps: 244927, Ep.Steps: 1465, Score: 266.51, Avg.Score: 57.07, Time: 04:08:37\n",
      "Ep.: 223, Total Steps: 246376, Ep.Steps: 1449, Score: 269.29, Avg.Score: 59.62, Time: 04:10:17\n",
      "Ep.: 224, Total Steps: 247751, Ep.Steps: 1375, Score: 270.73, Avg.Score: 62.33, Time: 04:12:01\n",
      "Ep.: 225, Total Steps: 249146, Ep.Steps: 1395, Score: 275.72, Avg.Score: 65.17, Time: 04:13:32\n",
      "Ep.: 226, Total Steps: 250566, Ep.Steps: 1420, Score: 271.56, Avg.Score: 67.91, Time: 04:15:06\n",
      "Ep.: 227, Total Steps: 251862, Ep.Steps: 1296, Score: 280.31, Avg.Score: 70.65, Time: 04:16:33\n",
      "Ep.: 228, Total Steps: 253161, Ep.Steps: 1299, Score: 278.66, Avg.Score: 73.43, Time: 04:17:56\n",
      "Ep.: 229, Total Steps: 254442, Ep.Steps: 1281, Score: 280.51, Avg.Score: 76.31, Time: 04:19:17\n",
      "Ep.: 230, Total Steps: 255823, Ep.Steps: 1381, Score: 273.08, Avg.Score: 79.35, Time: 04:20:45\n",
      "Ep.: 231, Total Steps: 257047, Ep.Steps: 1224, Score: 282.21, Avg.Score: 82.54, Time: 04:22:02\n",
      "Ep.: 232, Total Steps: 258463, Ep.Steps: 1416, Score: 274.71, Avg.Score: 85.63, Time: 04:23:31\n",
      "Ep.: 233, Total Steps: 259745, Ep.Steps: 1282, Score: 280.09, Avg.Score: 89.32, Time: 04:24:52\n",
      "Ep.: 234, Total Steps: 261057, Ep.Steps: 1312, Score: 277.73, Avg.Score: 93.01, Time: 04:26:16\n",
      "Ep.: 235, Total Steps: 262374, Ep.Steps: 1317, Score: 276.52, Avg.Score: 96.79, Time: 04:27:40\n",
      "Ep.: 236, Total Steps: 263740, Ep.Steps: 1366, Score: 277.00, Avg.Score: 100.51, Time: 04:29:11\n",
      "Ep.: 237, Total Steps: 265076, Ep.Steps: 1336, Score: 278.11, Avg.Score: 104.35, Time: 04:30:33\n",
      "Ep.: 238, Total Steps: 266397, Ep.Steps: 1321, Score: 278.99, Avg.Score: 107.69, Time: 04:31:54\n",
      "Ep.: 239, Total Steps: 267642, Ep.Steps: 1245, Score: 283.17, Avg.Score: 111.29, Time: 04:33:15\n",
      "Ep.: 240, Total Steps: 269008, Ep.Steps: 1366, Score: 278.90, Avg.Score: 114.93, Time: 04:34:57\n",
      "Ep.: 241, Total Steps: 270387, Ep.Steps: 1379, Score: 276.81, Avg.Score: 117.57, Time: 04:36:23\n",
      "Ep.: 242, Total Steps: 271741, Ep.Steps: 1354, Score: 278.23, Avg.Score: 120.75, Time: 04:37:50\n",
      "Ep.: 243, Total Steps: 272998, Ep.Steps: 1257, Score: 284.69, Avg.Score: 123.48, Time: 04:39:09\n",
      "Ep.: 244, Total Steps: 273263, Ep.Steps: 265, Score: -49.55, Avg.Score: 123.46, Time: 04:39:26\n",
      "Ep.: 245, Total Steps: 274579, Ep.Steps: 1316, Score: 280.63, Avg.Score: 127.79, Time: 04:40:50\n",
      "Ep.: 246, Total Steps: 275919, Ep.Steps: 1340, Score: 280.52, Avg.Score: 130.40, Time: 04:42:15\n",
      "Ep.: 247, Total Steps: 277191, Ep.Steps: 1272, Score: 283.78, Avg.Score: 133.74, Time: 04:43:35\n",
      "Ep.: 248, Total Steps: 278520, Ep.Steps: 1329, Score: 281.75, Avg.Score: 136.37, Time: 04:44:57\n",
      "Ep.: 249, Total Steps: 279862, Ep.Steps: 1342, Score: 279.98, Avg.Score: 139.10, Time: 04:46:21\n",
      "Ep.: 250, Total Steps: 281229, Ep.Steps: 1367, Score: 279.24, Avg.Score: 141.69, Time: 04:47:43\n",
      "Ep.: 251, Total Steps: 282519, Ep.Steps: 1290, Score: 282.20, Avg.Score: 144.28, Time: 04:48:59\n",
      "Ep.: 252, Total Steps: 283779, Ep.Steps: 1260, Score: 286.69, Avg.Score: 146.78, Time: 04:50:31\n",
      "Ep.: 253, Total Steps: 284989, Ep.Steps: 1210, Score: 288.90, Avg.Score: 149.27, Time: 04:51:47\n",
      "Ep.: 254, Total Steps: 286259, Ep.Steps: 1270, Score: 286.88, Avg.Score: 151.59, Time: 04:53:03\n",
      "Ep.: 255, Total Steps: 287495, Ep.Steps: 1236, Score: 290.50, Avg.Score: 154.20, Time: 04:54:19\n",
      "Ep.: 256, Total Steps: 288768, Ep.Steps: 1273, Score: 285.39, Avg.Score: 157.90, Time: 04:55:34\n",
      "Ep.: 257, Total Steps: 289990, Ep.Steps: 1222, Score: 287.60, Avg.Score: 160.06, Time: 04:56:46\n",
      "Ep.: 258, Total Steps: 291211, Ep.Steps: 1221, Score: 290.09, Avg.Score: 162.30, Time: 04:57:59\n",
      "Ep.: 259, Total Steps: 292436, Ep.Steps: 1225, Score: 286.70, Avg.Score: 164.17, Time: 04:59:11\n",
      "Ep.: 260, Total Steps: 293618, Ep.Steps: 1182, Score: 292.21, Avg.Score: 168.02, Time: 05:00:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 261, Total Steps: 294778, Ep.Steps: 1160, Score: 292.08, Avg.Score: 172.04, Time: 05:01:30\n",
      "Ep.: 262, Total Steps: 295946, Ep.Steps: 1168, Score: 290.61, Avg.Score: 176.00, Time: 05:02:38\n",
      "Ep.: 263, Total Steps: 297105, Ep.Steps: 1159, Score: 291.76, Avg.Score: 179.71, Time: 05:03:47\n",
      "Ep.: 264, Total Steps: 298300, Ep.Steps: 1195, Score: 291.77, Avg.Score: 182.65, Time: 05:04:57\n",
      "Ep.: 265, Total Steps: 299507, Ep.Steps: 1207, Score: 289.90, Avg.Score: 186.56, Time: 05:06:09\n",
      "Ep.: 266, Total Steps: 300783, Ep.Steps: 1276, Score: 284.74, Avg.Score: 190.30, Time: 05:07:24\n",
      "Ep.: 267, Total Steps: 301955, Ep.Steps: 1172, Score: 292.28, Avg.Score: 193.32, Time: 05:08:33\n",
      "Ep.: 268, Total Steps: 303117, Ep.Steps: 1162, Score: 289.38, Avg.Score: 196.71, Time: 05:09:42\n",
      "Ep.: 269, Total Steps: 304307, Ep.Steps: 1190, Score: 291.95, Avg.Score: 199.70, Time: 05:10:52\n",
      "Ep.: 270, Total Steps: 305443, Ep.Steps: 1136, Score: 293.81, Avg.Score: 202.92, Time: 05:12:00\n",
      "Ep.: 271, Total Steps: 306633, Ep.Steps: 1190, Score: 288.97, Avg.Score: 206.05, Time: 05:13:10\n",
      "Ep.: 272, Total Steps: 307847, Ep.Steps: 1214, Score: 289.57, Avg.Score: 209.45, Time: 05:14:22\n",
      "Ep.: 273, Total Steps: 309013, Ep.Steps: 1166, Score: 294.51, Avg.Score: 212.89, Time: 05:15:33\n",
      "Ep.: 274, Total Steps: 310245, Ep.Steps: 1232, Score: 289.53, Avg.Score: 216.39, Time: 05:16:45\n",
      "Ep.: 275, Total Steps: 311405, Ep.Steps: 1160, Score: 292.90, Avg.Score: 219.91, Time: 05:17:54\n",
      "Ep.: 276, Total Steps: 312511, Ep.Steps: 1106, Score: 296.72, Avg.Score: 223.35, Time: 05:19:00\n",
      "Ep.: 277, Total Steps: 313590, Ep.Steps: 1079, Score: 296.43, Avg.Score: 227.34, Time: 05:20:04\n",
      "Ep.: 278, Total Steps: 314683, Ep.Steps: 1093, Score: 296.70, Avg.Score: 230.47, Time: 05:21:08\n",
      "Ep.: 279, Total Steps: 315850, Ep.Steps: 1167, Score: 292.45, Avg.Score: 234.43, Time: 05:22:17\n",
      "Ep.: 280, Total Steps: 316984, Ep.Steps: 1134, Score: 295.58, Avg.Score: 237.96, Time: 05:23:24\n",
      "Ep.: 281, Total Steps: 318148, Ep.Steps: 1164, Score: 293.36, Avg.Score: 241.80, Time: 05:24:33\n",
      "Ep.: 282, Total Steps: 319295, Ep.Steps: 1147, Score: 291.40, Avg.Score: 243.87, Time: 05:25:41\n",
      "Ep.: 283, Total Steps: 320462, Ep.Steps: 1167, Score: 289.64, Avg.Score: 244.98, Time: 05:26:50\n",
      "Ep.: 284, Total Steps: 321567, Ep.Steps: 1105, Score: 292.53, Avg.Score: 246.28, Time: 05:27:55\n",
      "Ep.: 285, Total Steps: 322640, Ep.Steps: 1073, Score: 296.73, Avg.Score: 248.65, Time: 05:28:59\n",
      "Ep.: 286, Total Steps: 323415, Ep.Steps: 775, Score: 74.52, Avg.Score: 247.74, Time: 05:29:44\n",
      "Ep.: 287, Total Steps: 324524, Ep.Steps: 1109, Score: 294.37, Avg.Score: 248.61, Time: 05:30:50\n",
      "Ep.: 288, Total Steps: 325639, Ep.Steps: 1115, Score: 292.89, Avg.Score: 249.62, Time: 05:31:56\n",
      "Ep.: 289, Total Steps: 326708, Ep.Steps: 1069, Score: 295.89, Avg.Score: 250.81, Time: 05:32:59\n",
      "Ep.: 290, Total Steps: 327824, Ep.Steps: 1116, Score: 291.16, Avg.Score: 252.10, Time: 05:34:06\n",
      "Ep.: 291, Total Steps: 328902, Ep.Steps: 1078, Score: 295.53, Avg.Score: 253.22, Time: 05:35:10\n",
      "Ep.: 292, Total Steps: 329997, Ep.Steps: 1095, Score: 292.99, Avg.Score: 255.09, Time: 05:36:15\n",
      "Ep.: 293, Total Steps: 331109, Ep.Steps: 1112, Score: 293.84, Avg.Score: 256.45, Time: 05:37:20\n",
      "Ep.: 294, Total Steps: 332132, Ep.Steps: 1023, Score: 299.87, Avg.Score: 258.04, Time: 05:38:21\n",
      "Ep.: 295, Total Steps: 332780, Ep.Steps: 648, Score: 74.58, Avg.Score: 257.01, Time: 05:38:59\n",
      "Ep.: 296, Total Steps: 333914, Ep.Steps: 1134, Score: 292.56, Avg.Score: 258.19, Time: 05:40:07\n",
      "Ep.: 297, Total Steps: 334562, Ep.Steps: 648, Score: 55.28, Avg.Score: 256.84, Time: 05:40:45\n",
      "Ep.: 298, Total Steps: 335651, Ep.Steps: 1089, Score: 292.59, Avg.Score: 258.06, Time: 05:41:50\n",
      "Ep.: 299, Total Steps: 336719, Ep.Steps: 1068, Score: 293.33, Avg.Score: 259.21, Time: 05:42:53\n",
      "Ep.: 300, Total Steps: 337800, Ep.Steps: 1081, Score: 294.98, Avg.Score: 260.23, Time: 05:43:57\n",
      "Ep.: 301, Total Steps: 338883, Ep.Steps: 1083, Score: 293.93, Avg.Score: 261.13, Time: 05:45:02\n",
      "Ep.: 302, Total Steps: 339961, Ep.Steps: 1078, Score: 295.20, Avg.Score: 262.43, Time: 05:46:06\n",
      "Ep.: 303, Total Steps: 341043, Ep.Steps: 1082, Score: 294.30, Avg.Score: 265.35, Time: 05:47:10\n",
      "Ep.: 304, Total Steps: 342142, Ep.Steps: 1099, Score: 293.38, Avg.Score: 267.77, Time: 05:48:15\n",
      "Ep.: 305, Total Steps: 343142, Ep.Steps: 1000, Score: 299.39, Avg.Score: 268.09, Time: 05:49:14\n",
      "Ep.: 306, Total Steps: 344278, Ep.Steps: 1136, Score: 290.28, Avg.Score: 268.40, Time: 05:50:21\n",
      "Ep.: 307, Total Steps: 345386, Ep.Steps: 1108, Score: 293.34, Avg.Score: 268.78, Time: 05:51:26\n",
      "Ep.: 308, Total Steps: 346412, Ep.Steps: 1026, Score: 296.43, Avg.Score: 269.01, Time: 05:52:27\n",
      "Ep.: 309, Total Steps: 347542, Ep.Steps: 1130, Score: 290.95, Avg.Score: 269.22, Time: 05:53:34\n",
      "Ep.: 310, Total Steps: 348606, Ep.Steps: 1064, Score: 296.28, Avg.Score: 269.51, Time: 05:54:37\n",
      "Ep.: 311, Total Steps: 349628, Ep.Steps: 1022, Score: 295.52, Avg.Score: 269.79, Time: 05:55:38\n",
      "Ep.: 312, Total Steps: 350688, Ep.Steps: 1060, Score: 294.06, Avg.Score: 270.06, Time: 05:56:40\n",
      "Ep.: 313, Total Steps: 351791, Ep.Steps: 1103, Score: 291.16, Avg.Score: 270.28, Time: 05:57:45\n",
      "Ep.: 314, Total Steps: 352873, Ep.Steps: 1082, Score: 291.85, Avg.Score: 272.43, Time: 05:58:49\n",
      "Ep.: 315, Total Steps: 353923, Ep.Steps: 1050, Score: 295.31, Avg.Score: 275.89, Time: 05:59:51\n",
      "Ep.: 316, Total Steps: 354974, Ep.Steps: 1051, Score: 293.48, Avg.Score: 276.19, Time: 06:00:53\n",
      "Ep.: 317, Total Steps: 356042, Ep.Steps: 1068, Score: 293.07, Avg.Score: 276.54, Time: 06:01:57\n",
      "Ep.: 318, Total Steps: 357065, Ep.Steps: 1023, Score: 296.74, Avg.Score: 277.01, Time: 06:02:57\n",
      "Ep.: 319, Total Steps: 358088, Ep.Steps: 1023, Score: 297.66, Avg.Score: 277.55, Time: 06:03:57\n",
      "Ep.: 320, Total Steps: 359112, Ep.Steps: 1024, Score: 298.14, Avg.Score: 277.98, Time: 06:04:58\n",
      "Ep.: 321, Total Steps: 360149, Ep.Steps: 1037, Score: 295.72, Avg.Score: 278.33, Time: 06:05:59\n",
      "Ep.: 322, Total Steps: 361187, Ep.Steps: 1038, Score: 297.41, Avg.Score: 278.64, Time: 06:07:00\n",
      "Ep.: 323, Total Steps: 362192, Ep.Steps: 1005, Score: 298.99, Avg.Score: 278.94, Time: 06:08:00\n",
      "Ep.: 324, Total Steps: 363248, Ep.Steps: 1056, Score: 293.93, Avg.Score: 279.17, Time: 06:09:02\n",
      "Ep.: 325, Total Steps: 364269, Ep.Steps: 1021, Score: 298.19, Avg.Score: 279.39, Time: 06:10:03\n",
      "Ep.: 326, Total Steps: 365279, Ep.Steps: 1010, Score: 296.64, Avg.Score: 279.64, Time: 06:11:02\n",
      "Ep.: 327, Total Steps: 366303, Ep.Steps: 1024, Score: 297.09, Avg.Score: 279.81, Time: 06:12:02\n",
      "Ep.: 328, Total Steps: 367265, Ep.Steps: 962, Score: 302.17, Avg.Score: 280.05, Time: 06:12:59\n",
      "Ep.: 329, Total Steps: 368281, Ep.Steps: 1016, Score: 298.87, Avg.Score: 280.23, Time: 06:13:59\n",
      "Ep.: 330, Total Steps: 369271, Ep.Steps: 990, Score: 301.95, Avg.Score: 280.52, Time: 06:14:58\n",
      "Ep.: 331, Total Steps: 370256, Ep.Steps: 985, Score: 300.68, Avg.Score: 280.70, Time: 06:15:56\n",
      "Ep.: 332, Total Steps: 371255, Ep.Steps: 999, Score: 300.10, Avg.Score: 280.96, Time: 06:16:55\n",
      "Ep.: 333, Total Steps: 372239, Ep.Steps: 984, Score: 300.78, Avg.Score: 281.16, Time: 06:17:53\n",
      "Ep.: 334, Total Steps: 373209, Ep.Steps: 970, Score: 303.07, Avg.Score: 281.42, Time: 06:18:50\n",
      "Ep.: 335, Total Steps: 374203, Ep.Steps: 994, Score: 298.70, Avg.Score: 281.64, Time: 06:19:49\n",
      "Ep.: 336, Total Steps: 375230, Ep.Steps: 1027, Score: 294.80, Avg.Score: 281.82, Time: 06:20:49\n",
      "Ep.: 337, Total Steps: 376251, Ep.Steps: 1021, Score: 296.11, Avg.Score: 282.00, Time: 06:21:50\n",
      "Ep.: 338, Total Steps: 377269, Ep.Steps: 1018, Score: 295.93, Avg.Score: 282.17, Time: 06:22:50\n",
      "Ep.: 339, Total Steps: 378270, Ep.Steps: 1001, Score: 298.64, Avg.Score: 282.32, Time: 06:23:49\n",
      "Ep.: 340, Total Steps: 379226, Ep.Steps: 956, Score: 303.41, Avg.Score: 282.57, Time: 06:24:46\n",
      "Ep.: 341, Total Steps: 380199, Ep.Steps: 973, Score: 301.15, Avg.Score: 282.81, Time: 06:25:44\n",
      "Ep.: 342, Total Steps: 381162, Ep.Steps: 963, Score: 300.62, Avg.Score: 283.03, Time: 06:26:41\n",
      "Ep.: 343, Total Steps: 382143, Ep.Steps: 981, Score: 300.16, Avg.Score: 283.19, Time: 06:27:39\n",
      "Ep.: 344, Total Steps: 383117, Ep.Steps: 974, Score: 299.96, Avg.Score: 286.68, Time: 06:28:36\n",
      "Ep.: 345, Total Steps: 384130, Ep.Steps: 1013, Score: 296.46, Avg.Score: 286.84, Time: 06:29:36\n",
      "Ep.: 346, Total Steps: 385101, Ep.Steps: 971, Score: 299.50, Avg.Score: 287.03, Time: 06:30:33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 347, Total Steps: 386092, Ep.Steps: 991, Score: 300.15, Avg.Score: 287.19, Time: 06:31:33\n",
      "Ep.: 348, Total Steps: 387054, Ep.Steps: 962, Score: 302.13, Avg.Score: 287.40, Time: 06:32:46\n",
      "Ep.: 349, Total Steps: 388016, Ep.Steps: 962, Score: 299.23, Avg.Score: 287.59, Time: 06:33:46\n",
      "Ep.: 350, Total Steps: 389016, Ep.Steps: 1000, Score: 296.63, Avg.Score: 287.76, Time: 06:34:47\n",
      "Ep.: 351, Total Steps: 390025, Ep.Steps: 1009, Score: 299.28, Avg.Score: 287.94, Time: 06:35:58\n",
      "Ep.: 352, Total Steps: 390963, Ep.Steps: 938, Score: 303.66, Avg.Score: 288.11, Time: 06:37:11\n",
      "Ep.: 353, Total Steps: 391914, Ep.Steps: 951, Score: 301.67, Avg.Score: 288.23, Time: 06:38:17\n",
      "Ep.: 354, Total Steps: 392882, Ep.Steps: 968, Score: 300.39, Avg.Score: 288.37, Time: 06:39:15\n",
      "Ep.: 355, Total Steps: 393895, Ep.Steps: 1013, Score: 296.77, Avg.Score: 288.43, Time: 06:40:19\n",
      "Ep.: 356, Total Steps: 394852, Ep.Steps: 957, Score: 301.88, Avg.Score: 288.60, Time: 06:41:18\n",
      "Ep.: 357, Total Steps: 395841, Ep.Steps: 989, Score: 299.95, Avg.Score: 288.72, Time: 06:42:17\n",
      "Ep.: 358, Total Steps: 396789, Ep.Steps: 948, Score: 302.28, Avg.Score: 288.84, Time: 06:43:13\n",
      "Ep.: 359, Total Steps: 397727, Ep.Steps: 938, Score: 304.62, Avg.Score: 289.02, Time: 06:44:08\n",
      "Ep.: 360, Total Steps: 398673, Ep.Steps: 946, Score: 303.46, Avg.Score: 289.13, Time: 06:45:05\n",
      "Ep.: 361, Total Steps: 399627, Ep.Steps: 954, Score: 302.58, Avg.Score: 289.24, Time: 06:46:01\n",
      "Ep.: 362, Total Steps: 400546, Ep.Steps: 919, Score: 304.45, Avg.Score: 289.38, Time: 06:46:57\n",
      "Ep.: 363, Total Steps: 401520, Ep.Steps: 974, Score: 299.91, Avg.Score: 289.46, Time: 06:47:55\n",
      "Ep.: 364, Total Steps: 402504, Ep.Steps: 984, Score: 300.82, Avg.Score: 289.55, Time: 06:48:53\n",
      "Ep.: 365, Total Steps: 403448, Ep.Steps: 944, Score: 300.74, Avg.Score: 289.66, Time: 06:49:49\n",
      "Ep.: 366, Total Steps: 404398, Ep.Steps: 950, Score: 300.41, Avg.Score: 289.81, Time: 06:50:45\n",
      "Ep.: 367, Total Steps: 405332, Ep.Steps: 934, Score: 303.18, Avg.Score: 289.92, Time: 06:51:40\n",
      "Ep.: 368, Total Steps: 406284, Ep.Steps: 952, Score: 303.04, Avg.Score: 290.06, Time: 06:52:37\n",
      "Ep.: 369, Total Steps: 407225, Ep.Steps: 941, Score: 301.92, Avg.Score: 290.16, Time: 06:53:33\n",
      "Ep.: 370, Total Steps: 408135, Ep.Steps: 910, Score: 304.95, Avg.Score: 290.27, Time: 06:54:27\n",
      "Ep.: 371, Total Steps: 409086, Ep.Steps: 951, Score: 302.84, Avg.Score: 290.41, Time: 06:55:24\n",
      "Ep.: 372, Total Steps: 410079, Ep.Steps: 993, Score: 301.11, Avg.Score: 290.52, Time: 06:56:22\n",
      "Ep.: 373, Total Steps: 411096, Ep.Steps: 1017, Score: 298.42, Avg.Score: 290.56, Time: 06:57:22\n",
      "Ep.: 374, Total Steps: 412064, Ep.Steps: 968, Score: 299.36, Avg.Score: 290.66, Time: 06:58:20\n",
      "Ep.: 375, Total Steps: 413038, Ep.Steps: 974, Score: 301.34, Avg.Score: 290.75, Time: 06:59:17\n",
      "Ep.: 376, Total Steps: 414008, Ep.Steps: 970, Score: 302.98, Avg.Score: 290.81, Time: 07:00:15\n",
      "Ep.: 377, Total Steps: 414965, Ep.Steps: 957, Score: 301.98, Avg.Score: 290.86, Time: 07:01:11\n",
      "Ep.: 378, Total Steps: 415906, Ep.Steps: 941, Score: 304.06, Avg.Score: 290.94, Time: 07:02:07\n",
      "Ep.: 379, Total Steps: 416848, Ep.Steps: 942, Score: 301.29, Avg.Score: 291.03, Time: 07:03:02\n",
      "Ep.: 380, Total Steps: 417823, Ep.Steps: 975, Score: 301.21, Avg.Score: 291.08, Time: 07:04:00\n",
      "Ep.: 381, Total Steps: 418775, Ep.Steps: 952, Score: 301.71, Avg.Score: 291.17, Time: 07:04:56\n",
      "Ep.: 382, Total Steps: 419722, Ep.Steps: 947, Score: 303.31, Avg.Score: 291.28, Time: 07:05:52\n",
      "Ep.: 383, Total Steps: 420664, Ep.Steps: 942, Score: 302.03, Avg.Score: 291.41, Time: 07:06:48\n",
      "Ep.: 384, Total Steps: 421601, Ep.Steps: 937, Score: 302.32, Avg.Score: 291.51, Time: 07:07:43\n",
      "Ep.: 385, Total Steps: 422540, Ep.Steps: 939, Score: 303.65, Avg.Score: 291.58, Time: 07:08:39\n",
      "Ep.: 386, Total Steps: 423511, Ep.Steps: 971, Score: 301.58, Avg.Score: 293.85, Time: 07:09:37\n",
      "Ep.: 387, Total Steps: 424432, Ep.Steps: 921, Score: 303.88, Avg.Score: 293.94, Time: 07:10:31\n",
      "Ep.: 388, Total Steps: 425361, Ep.Steps: 929, Score: 302.59, Avg.Score: 294.04, Time: 07:11:27\n",
      "Ep.: 389, Total Steps: 426326, Ep.Steps: 965, Score: 301.26, Avg.Score: 294.09, Time: 07:12:23\n",
      "Ep.: 390, Total Steps: 427265, Ep.Steps: 939, Score: 304.71, Avg.Score: 294.23, Time: 07:13:19\n",
      "Ep.: 391, Total Steps: 428212, Ep.Steps: 947, Score: 302.76, Avg.Score: 294.30, Time: 07:14:15\n",
      "Ep.: 392, Total Steps: 429163, Ep.Steps: 951, Score: 303.86, Avg.Score: 294.41, Time: 07:15:12\n",
      "Ep.: 393, Total Steps: 430101, Ep.Steps: 938, Score: 303.55, Avg.Score: 294.51, Time: 07:16:07\n",
      "Ep.: 394, Total Steps: 431040, Ep.Steps: 939, Score: 303.91, Avg.Score: 294.55, Time: 07:17:02\n",
      "Ep.: 395, Total Steps: 431988, Ep.Steps: 948, Score: 302.51, Avg.Score: 296.83, Time: 07:17:59\n",
      "Ep.: 396, Total Steps: 432937, Ep.Steps: 949, Score: 304.57, Avg.Score: 296.95, Time: 07:18:55\n",
      "Ep.: 397, Total Steps: 433858, Ep.Steps: 921, Score: 305.46, Avg.Score: 299.45, Time: 07:19:50\n",
      "Ep.: 398, Total Steps: 434802, Ep.Steps: 944, Score: 304.52, Avg.Score: 299.57, Time: 07:20:46\n",
      "Ep.: 399, Total Steps: 435723, Ep.Steps: 921, Score: 305.65, Avg.Score: 299.69, Time: 07:21:40\n",
      "Ep.: 400, Total Steps: 436684, Ep.Steps: 961, Score: 303.97, Avg.Score: 299.78, Time: 07:22:38\n",
      "Ep.: 401, Total Steps: 437608, Ep.Steps: 924, Score: 306.29, Avg.Score: 299.90, Time: 07:23:33\n",
      "Ep.: 402, Total Steps: 438541, Ep.Steps: 933, Score: 304.28, Avg.Score: 299.99, Time: 07:24:28\n",
      "Ep.: 403, Total Steps: 439491, Ep.Steps: 950, Score: 304.15, Avg.Score: 300.09, Time: 07:25:24\n",
      "Ep.: 404, Total Steps: 440417, Ep.Steps: 926, Score: 305.15, Avg.Score: 300.21, Time: 07:26:19\n",
      "Ep.: 405, Total Steps: 441320, Ep.Steps: 903, Score: 305.71, Avg.Score: 300.27, Time: 07:27:13\n",
      "Ep.: 406, Total Steps: 442235, Ep.Steps: 915, Score: 306.88, Avg.Score: 300.44, Time: 07:28:07\n",
      "Ep.: 407, Total Steps: 443146, Ep.Steps: 911, Score: 305.48, Avg.Score: 300.56, Time: 07:29:01\n",
      "Solved environment with Avg Score:   300.56121912236614\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from  collections  import deque\n",
    "\n",
    "def sac_train(max_steps):\n",
    "\n",
    "    total_numsteps = 0\n",
    "    updates = 0\n",
    "    num_episodes = 10001\n",
    "    updates=0\n",
    "\n",
    "    time_start = time.time()\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = [] \n",
    "    \n",
    "    for i_episode in range(num_episodes): \n",
    "        episode_reward = 0\n",
    "        episode_steps = 0\n",
    "        done = False\n",
    "        state = env.reset()\n",
    "\n",
    "        for step in range(max_steps):    \n",
    "            if start_steps > total_numsteps:\n",
    "                action = env.action_space.sample()  # Sample random action\n",
    "            else:\n",
    "                action = agent.select_action(state)  # Sample action from policy\n",
    "\n",
    "            if len(memory) > batch_size:\n",
    "                \n",
    "                # Update parameters of all the networks\n",
    "                agent.update_parameters(memory, batch_size, updates)\n",
    "\n",
    "                updates += 1\n",
    "\n",
    "            next_state, reward, done, _ = env.step(action) # Step\n",
    "            episode_steps += 1\n",
    "            total_numsteps += 1\n",
    "            episode_reward += reward\n",
    "\n",
    "            mask = 1 if episode_steps == env._max_episode_steps else float(not done)\n",
    "\n",
    "            memory.push(state, action, reward, next_state, mask) # Append transition to memory\n",
    "\n",
    "            state = next_state\n",
    "            \n",
    "            if done:\n",
    "                break\n",
    "\n",
    "        scores_deque.append(episode_reward)\n",
    "        scores_array.append(episode_reward)        \n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "        \n",
    "        if i_episode % 20 == 0 and i_episode > 0:\n",
    "            save(agent, 'dir_chk_lr00008', 'weights', str(i_episode))\n",
    "\n",
    "        s =  (int)(time.time() - time_start)\n",
    "            \n",
    "        print(\"Ep.: {}, Total Steps: {}, Ep.Steps: {}, Score: {:.2f}, Avg.Score: {:.2f}, Time: {:02}:{:02}:{:02}\".\\\n",
    "            format(i_episode, total_numsteps, episode_steps, episode_reward, avg_score, \\\n",
    "                  s//3600, s%3600//60, s%60))\n",
    "\n",
    "                    \n",
    "        if (avg_score > 300.5):\n",
    "            print('Solved environment with Avg Score:  ', avg_score)\n",
    "            break;\n",
    "            \n",
    "    return scores_array, avg_scores_array \n",
    "\n",
    "scores, avg_scores = sac_train(max_steps=max_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(agent, 'dir_chk_lr00008', 'weights', 'final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  408 , len of avg_scores:  408\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAEGCAYAAACtn3UnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXyU1dXA8d+dJftOwg4JAmGRTURUBAVxF1Fxt7Vq66ut1Vbr1mpfba2tVku1WqVqXVEriuurSBWUTUQBkX0RJGEJkH1PZr3vH7NkZjKTTCYzGQLn+/n0MzPPPPM8N1SYk3POvVdprRFCCCGE6C4M8R6AEEIIIURHSPAihBBCiG5FghchhBBCdCsSvAghhBCiW5HgRQghhBDdiineA4il3NxcXVBQEO9hCCFEt7J27dpyrXVevMchRChHdPBSUFDAmjVr4j0MIYToVpRSxfEegxBtkbKREEIIIboVCV6EEEII0a1I8CKEEEKIbuWI7nkRQghx+Fi7dm1Pk8n0b2AU8suzCM0JbLLb7Tccf/zxpcFOkOBFCCFElzCZTP/u3bv3iLy8vCqDwSAb64mgnE6nKisrG3nw4MF/AzODnSORrxBCiK4yKi8vr1YCF9EWg8Gg8/LyanBl6IKf04XjEUIIcXQzSOAiwuH+7yRkjCLBixBCdHPFFQ1o7YoJvj9Ux5vf7MHucMZ5VELETtx6XpRSScAyINE9jvla6weUUoOAN4Ec4FvgGq21VSmVCLwKHA9UAFdorYviMnghxFHLandiNiqUUt5jm/bXsKWklukjerK2uIozR/bye7+ovIH/fLOHX00fSmpiyz+7NoeTPZWNDM5L87tHs82BU2ssNifNdgefbDzIuaN7Y7E5MRkVX2wvY97qPdQ22dlT2QjA+aP7YHc6+XTLIbSGBZsO8vJ1J2AwKIS/e+65p/c777zTw2AwaIPBwDPPPFN8+umnN8R7XCJ88WzYtQCna63rlVJmYIVS6hPgN8DjWus3lVL/An4GzHE/VmmthyilrgT+ClwRr8ELIbqXNUWVmIwGappsTBrcg437a/jHou957NIx9MxIAuCmuWtITTQx+7KxfsHHN7sryUlNYHBeKrPmfEmSychrN5zIwZpmGqx2rn3xGyoarN7zX/vZiUwemss/Fn3PvNV7KKlpBmBfdRP/vOo4mm1O7nlnAx+uLwHg6avHU1TRwLzVezluYBYffOc6npZoomd6Ij+UN/DgR1v8fp4eqQnUNdu9rz/eeID0JBM3TB5EdmoCZoNBApcgFi1alPrf//43a+PGjVuSk5P1gQMHTBaLJeI/KJvNhtlsjuYQRRjiFrxoV46z3v3S7P6fBk4HrnYffwX4A67g5UL3c4D5wD+VUkp7cqVCiG7J4dS8v24/M8f1xWwMXskurmigV0YSSWZjh65tczgxGw3YHU4u/ddX3uPH5KZSVNGAU8OrXxVz59nD2FVWz383HwLgu73VnHRMD66YMIA1xVX86aMtJBgNPPOj8WzaXwvAJXNWsrnE9dxkUPzpwmOZ/+1+1u+t5scvfE1hrzR2ltaTmWzGoOCkY3rw8YYDFPZMZ01xJcu/L/eO55dvfOt9vreq0fu8b1YSOw7Vk5Fk4rzRfRjVL5PVRZUYFTx6Qj2m2r3UVJWT6qyjqa6SJHsd5to6OFQNPY4Bnu7Qn9fRYP/+/eacnBx7cnKyBujTp48dYOnSpSm33XbbwMbGRkNCQoJetmzZ9sTERP2Tn/wkf8OGDSlGo5FHH3107wUXXFD35JNP9vjkk08yLRaLobGx0bBq1aod//u//9vrvffey7Farer888+vfvzxx0vi+5Me2eI6VVopZQTWAkNw/S3bBVRrrT2/TuwD+rmf9wP2Amit7UqpGqAHUB5wzRuBGwEGDhwY6x9BCOHDanfSYLGjFGSlJPi912R1kGQ2+GU0AOYs2cnfPt2BwQAXH9e/1TVrm22c9tgSLhrXlyeuPK7N+6/cVc7ba/YxvHc6q4uq2LS/hilDc2m2+/d/pCWZmDgoh5LqZt5cvYdbpw/hja/3YFDw6+mFrCmu5I2v9/DG13sASDAasDqc3PDqGjKTzfz+/BHcNX+D93qzLx/LheP6cc3JBTz+2Q7+b0MJBT1SGZiTwuzLx2E2KpLNRm55Yx2PL9oBwKOXjGHmuL5UNFhZvPUQg3JTOWVwLjanE6NS7K1qol9WMku2l3LasDwSTa7A7cfH94Lls+HVRwHIdI8hPTEDkrIgKROSsyAxo53/t+LrrvnrB+w4WJcSzWsW9k5vfOzSsXvbOueiiy6qffjhh/sWFBSMmjx5cu1VV11VOX369IYf/ehHg19//fVdp512WmNlZaUhLS3N+dBDD/UC2LFjx5Z169YlnXfeeUN37dq1CeDbb79N27Bhw+ZevXo53n333YydO3cmbdiwYavWmjPOOGPIJ598knbuuefWtzUWEbm4Bi9aawcwTimVBbwHjAh2mvsxWFqvVdZFa/0c8BzAhAkTJCsjRBeoabKRbDZy8+vfsmirK3txzUn5jO6fyeUTBtBkdTDi/oX8Yupgbpk2hIc+3srgvFSarA5v6SQ9sSX1/v2hOioarOwsrfc2nn625RA/n7uW0rpmqhptnDGiJ3eePYzSWguriyrJTkng+pdXtxrb22v3AZCTmsBfLxnDxEE5ZCa77rV0RxnXvvgNw36/EIDLJ/Tn12cMBeCL7aVc/9JqJhbkMO+mk7h7/gbeXruPu84exmUTBvDXhdspr7ew7U/n+GWEbj+zkNvPLAz65/TopWPon53MacPymDQ4F4B+Wcn85OQC7zmJBte1BuWmAnDWiDyw1EFDPVjq4b2b4MB3cMxUmPGEK1hJygRDx7JSR6vMzEznpk2btixcuDB98eLF6ddee+3g22+//UDPnj1tp512WiNATk6OE2DlypVpt956aynAcccd19y3b1/rxo0bkwCmTJlS26tXLwfAwoULM5YtW5YxcuTIkQCNjY2Gbdu2JUnwEjuHxSJ1WutqpdQS4CQgSyllcmdf+gOe1Ns+YACwTyllwvULR2U8xivE0aLZ5mBfVSODctPwtE84NZTWNVPTZOOBDzbzvzNGMuOpFRT2SmPHoXqG9Upn+6E65q5ybUxsMijsDtfvEXOW7GLOkl1B71VaZ2FfVSNfbC/jzx9vodnmny1psDpYuPmg9/Xzy3dTXNHIvqomthxwlW/6Zyfzf7dM5oEPN3P68J7cNu87AJ750XjGDciib1ay3zWnDMllWK909lY1csaIXtx33kjve1ML83j2muM5aVAPlFL86aJRzBzXl1PcQcfC26ZQ2WDtUCkrNdHE784L9jtagC0fwif3QFMV2Jtav3/mgzDxJjAnhX3vw017GZJYMplMzJgxo27GjBl1Y8aMafrXv/6Vp5QK9stwyGukpKQ4fc+77bbbDtx1113lIT8goiqes43yAJs7cEkGzsDVhPsFcCmuGUfXAh+4P/Kh+/VX7vc/l34XIaKrrtnG0h1lnD+6D8UVjTy+aIe3edRoUIzql8m0YXk8seh772euffEbAG9vxvxfnMy/l+/mH4u/Z3S/TH7z1vqw7n3vexu9z3ukJnDNSf04Pj+HL7aVsmJnOfurm8hIMlHrblK955zh/HXhNgBSE4xkpyZw19nDyE5N4MmrXOWlzGQziWaDN8sRyGBQzP/FyZiNhlZBiFKKs4/t7X2dZDYyZWie93VuWiK5aYnBfxhbsytbYql1P4Z43ux7rA4sNa7Hmv2QNwxGzYLEdEhIg8Q012OPwdC37fKZCG39+vWJBoOB0aNHWwDWrVuXPHTo0OYlS5ZkLl26NOW0005rrKqqMqSlpTknT55c/9prr+XMnDmzbsOGDYkHDhxIGDNmTPPXX3/tV+4699xza//whz/0vfHGGyszMzOdu3fvNickJOh+/frZg49CdFY8My99gFfcfS8G4C2t9UdKqS3Am0qph4B1wAvu818A5iqlduLKuFwZj0ELcSRyOjV/XrCVF1bsBmDNpCpeXlkEuGa8JJmNjBuQxdIdpazfW+393NCeaXxf2pIZv+6UQaQnmfnV9KFcfeJAEowGzn5iGaV1Fr/7fX3vdB78aAv3nTeCygYrM55a4ff+k1cdxylDXAHHOaN6s2l/DTP/uYLHrxjHvNV7uWriQCYPzWV3eT19MpO55fQhQZt9pw3v2e7Pnp4UxkwRWxM0lENDGTRWuB6Dvi6HxnKwNbZ/TYMZkjJcwUliuqtHJaOf63FYHpzya0jv1f51RIfU1tYaf/WrXw2sra01Go1GXVBQYHnllVeKd+zYUf6rX/1qYHNzsyEpKcm5bNmyHXfffXfpNddck19YWDjSaDTy7LPPFnkafX3NmjWrdvPmzUknnHDCcHBlZV5//fXdErzEjjqSkxcTJkzQa9asifcwhDjsPbt0Fw9/si3oe6/8dCKnFboyDrfP+4731u1nYE4KD154LH2zknl26Q9cOXEA+6uaOPvY3iQn+GcwqhqsbC6p5Zb/fEt1ow2A7/98rjfY2FJSy3lPLvf7zBv/c2KrbEm9xU5aYhf+vrXoD7DpXVdwYg3RumBMhNQ8SO3hekzJhdRcSM529aH4BiaBzw/jko9Saq3WekK0r7t+/fqisWPHSmlFhGX9+vW5Y8eOLQj23mHR8yKEiA2HU7P1QC2j+mWyZHspd7y1nqV3T8Nic/D+dyX86MSBmAyKF7/czcSCHKYOz2PJ9jJ2HKrj6avH8/m2UiYN7uG93rThPXlv3X7OHd2bqcNcWY3Zl48F4ISC4GPITk1g8tBcDD6zjPyeB5kdbVCt+/O7JHBxOqDuAOxbDSseh4IpMHxGQHDiE6wkpEGQsQohYkuCFyG6Ma01dqdmx6E6HvhgM/++doLfFOV/fr6Txxft4O2fn8yt/1lHXbOdLSW1PLFoByt3VbCmqJKZY/tyqNbCH2eO4pxRvbnmpHyarA56ZiR5Szce54/uQ22TjYuO6xc4lHb5rpfm/7z1l3+wY1Fht0DNPqjZC9V7Wx6r90DNHqgtAac705/aE658w1XaEUIcViR4EaKbWltcxSVzVvodG/fgZ1x94kD+fNEo6i12Xv/aNePnja/3eFdjve6lb2i0Oji2bwafbDrIJ5sOckxuKtNHuDIp6UnmkH0gRoPixyflRzRe3/VdlF8WpvW5nVoY1mGDip1QugVKt0LlDy2BSt1B/FdYUJDRFzIHwIATXY9ZAyBzoKspVgIXIQ5LErwI0Q3ZHU4Wu9dTCfTG13s4WNPMhn3VlNdbGZCTzHvr9nvfb7Q6yEox884vJvH2mr288c1e7p8xMuTqttHiCUgCA5PARetCHQvJ6YQ9X7lKPXu/ht3LWnpUlNEVjGQNhMHT3YHJgJbHjH5gSmj7+kKIw44EL0LEWUW9haKKRo7Pzw77MzOeWsG2g3UM7ZnG5RMG8OcFW73vTRmay/Lvy5iQn8OL1w2nsFc6N85dy6ofKrC6V5o9Y0QvksxGrjm5gGt8FkiLJU8pKLAkFLxs1IELL7ofVj7lep49CEZfBvmToOdIyB0KphDTmYUQ3ZYEL0LE2Z1vr+eL7WV8c+907waBbfn+UB3bDtYB0DMjkUlDWhpqr5o4gIdnjcHp1H6b8r3604n83/oSbv3POiD4ctWxFjp4CX1uu6r3wNfPwahL4NxHXTN9hBBHvNjmiYUQ7Sqvd+1GPP9b1zL2VrsThzP4EgY2h9Nvgbhpw3oysk8Gf7l4NGt+fwYPzxoDEHQ3YYNfz0nUhh82zz0D7x12w67WUHsAdn0Bq/4FH/0GXpkJxgSY/oAELiJsr776apZS6vh169YdFvPVDx48aDzxxBMLU1JSjvvJT37itynf8uXLUwoLC0cOHDhw1HXXXTfA6XRlTw8dOmScNGnS0Pz8/FGTJk0aWlZWFpX9IU477bQh5eXlnbrWRx99lD5t2rQh0RhPKBK8CBEna4ur2LivhvQkVwL00YXbueLZryj8/Sf87t0NQT/zz8938vHGA9xxZiGr7zuD608ZhFKKq08cGHq1VzfVzgyfWAuVeQk2FL9j3/0HXjgL/poPfx8Ocy+ChffApvmQkgNXvArZkTURi6PTm2++mTN+/Pj6uXPn5sR7LAApKSn6wQcfLPnDH/6wL/C9m2++Of+ZZ54pLioq2vTDDz8kzZ8/PwPggQce6DN16tS64uLiTVOnTq27//77e7e+csctXbp0Z25uriMa14olCV6EiJNL5qzkgn+uoKzOwtj+mRyTm8o69+q1n28ra3V+VYOVOUt2MXNsX26dPpS89ESMHWgO8T0zHpmXUA27wQIp76EDG+CDX7qW0R91KZz7GPzkQ7hjB9xTDP/zOQw+PbYDF0eUmpoaw5o1a9Jeeumlovfee8/baHb++ecfM2/ePM8m3VxyySUFL7/8clZdXZ3hvPPOO6awsHDk+eeff8yYMWOGL1u2rNVu2B988EH6iBEjRhYWFo687LLLCpqamhRAv379Rt9+++19R44cOaKwsHBksGxPRkaG8+yzz65PSkry29CruLjYXF9fbzjjjDMaDAYDP/rRjyref//9bICFCxdm3XTTTRUAN910U8Unn3zSqmnObrdz00039R81atSIwsLCkY899lguuDIjEyZMGHbmmWcOHjx48LFXX331QIfDFa/069dv9IEDB0y1tbWGqVOnDhk2bNjIoUOHHvv8889nt/Vzzp8/P2PQoEHHHn/88cPmz5+f5RlDbW2t4bLLLisYNWrUiBEjRox87bXXsgDWrFmTNHr06BHDhw8fWVhYOHLjxo0dak6Tnhch4mx/dROXjO/Pny4aRZPVwbS/LaGwdzr7qhr5/lC9d4n7L7aXYnU4uWHKoIjuE2qqclfpWMOuArvVFbik9IDrF7iyLOLI8f4vB1C6pVUQ0Ck9RzZy0dNtbvj4+uuvZ02dOrVmzJgxlqysLMeKFStSJk+e3HjFFVdUzps3L/uKK66oaW5uVl9++WXGK6+8Uvzoo4/2zMrKcuzYsWPL6tWrk04++eRjA6/Z2NiobrrppkGffvrp9jFjxlguvvjigsceeyzv/vvvLwXIzc21b9myZesjjzyS98gjj/SaN29ecTg/TnFxsblPnz42z+v8/HzrgQMHzAAVFRWm/Px8m/u4rbKystX3+RNPPJGbmZnp2LRp09ampiZ1wgknDL/gggtqATZu3Ji6bt26TYWFhdZTTz116Kuvvpp9/fXXV3k+++6772b07t3btmTJkp3u+xlD/Zx33nln2S233FLw2WefbT/22GMtM2bMOMZznXvvvbfPtGnTat9+++2i8vJy44QJE0bMnDmz9qmnnsq7+eabD/3iF7+obG5uVnZ7x3ZSkMyLEHHWaHV4Sz7JCUb6ZiWhteb2ed9x/curOVTbDMDibaXkpScyqm9mW5cLyTdGiEfDbuiel9bnph5cDS+cAQc3wPmzJXARUfPWW2/lXHXVVVUAl1xySaWndHTppZfWrFy5MqOpqUnNnz8/c+LEiXVpaWl65cqVaVdddVUlwAknnNBcWFjYauOq9evXJ/Xv398yZswYC8B1111XsWLFinTP+1dffXUVwMSJExv37t0bdoYh2PY9HfnFY9GiRRlvvfVWj+HDh4887rjjRlRVVZm2bNmSBDB69OiGkSNHWk0mE5dffnnl8uXL03w/O378+Kbly5dn/OIXv+i3cOHCtB49ejhC/ZzfffddUv/+/S2jR4+2eDJEnussWbIk4/HHH+8zfPjwkZMnTx5msVjUzp07E04++eSG2bNn97nvvvt6f//99wlpaWkd2qtIMi9CxEmP1AQqGlzNunnpLf+eGZTC4dTe5fDf/XY/N0wZxLLtZZw3uk/QZtxwxL9s5M68BIxf2Ru52rgYA07qdDInGLYz8IPFrjVYLn8VRs7s+sGK2GsnQxILBw8eNK5atSpjx44dybfccgsOh0MppfScOXP2paSk6JNOOqnu3XffzZg3b162J2AJZ/+/9s5JSkrSACaTSdvt9rD/9hUUFNg8mRaA4uLihN69e9sAevToYS8uLjbn5+fbiouLzTk5Oa1SF1prNXv27D2XXHJJre/xjz76KD0wCAp8PWbMGMu333675Z133sm87777+i1atKh21qxZ1YQQKqjSWjN//vydY8eO9duddfz48c1TpkxpeO+99zLPPffcwmeeeaZo5syZdSH/MAJI5kWIOPGdFp2T2rJQmsGgcGpNfo9UABZvPcTqokrqLHZOH9H+LsmhhNpbqKt47mn0vXdtCdlvzuQv5hd4yPwS/0h4hquMn1M19ka4ZQ2MvLDLxymOXHPnzs2eNWtWRUlJycb9+/dvPHjw4Ib+/ftbP/300zSAK6+8svLll1/OXb16dfqsWbNqASZNmlT/5ptvZgOsXbs2aceOHcmB1x03blzz/v37EzZt2pQI8Oqrr/aYMmVK2F/EoeTn59tSU1OdixcvTnU6nbz++us9LrzwwmqAs88+u/rZZ5/tAfDss8/2OOecc1oFFmeeeWbNnDlz8iwWiwLYsGFDYm1trQFcZaNt27YlOBwO5s+fnxM43qKiInN6errz5ptvrrztttsOfffddymhfs5x48Y179u3L2Hz5s2J4GqI9lxn2rRptbNnz+7lmSX15ZdfJgNs2bIlYcSIEZbf//73pWeddVb1d9991+rPtS2SeREiTnx/W8vv0VL6NyjwnSm9qaSGz7eWYjYqJg+JfDrw4VM2cj+p2Qcvn4+xvow/2q6hRqeSoixscB7DE5N/SnZCdNshhHj77bd73H333Qd8j1144YVVc+fOzTnnnHPqL7744tqf//zng84444xqT7bkrrvuKrv88ssLCgsLR44aNapx2LBhTdnZ2X6zcVJSUvS//vWvossuu2yww+Fg7NixjXfeeWfrrvs29OvXb3R9fb3RZrOp//73v1kLFizYcfzxxzc/88wzxT/72c8GNTc3q2nTptVedtllNQB//OMfD1x88cWD8/Pzc/v27Wt9//33dwVe8/bbby8vKipKHD169AittcrJybEtWLBgF8C4cePq77jjjv7btm1LPvHEE+uuueYav+Bn7dq1yb/73e/6GwwGTCaTfuaZZ4pD/ZzJycn6qaeeKp4xY8aQnJwc+4knnli/devWZIBHHnmk5MYbbxw4fPjwkVpr1b9/f8sXX3yxc+7cuTlvv/12D5PJpPPy8mwPP/xwSUf+vFQ4KbHuasKECXrNmjXxHoYQQZ31+FIG56Xx4IWj/MpGVz+/Cqvdyci+Gbz6lauvLy89kby0RBb8ekrE9/t82yF++rLr78N1kwr4w8xWfYcxdd4/lrPlQC090xP5ZmY1fHwHOO3UX/Euo54v9zt3yZ1TKchN7dLxiRZKqbVa6wnRvu769euLxo4dW97+mYcPu92O1WpVKSkpevPmzYlnnXVW4a5duzZ5gpvu6KOPPkqfPXt2ry+++GJnvMfSlvXr1+eOHTu2INh7knkRIk4cTo1BKb/ABVybHzq1xvf3irI6C6cM7kFn+M826tSlIjLBvpaBhiqutK+Ed76BfhNg1nM4UwcCn/qdG4/xCRFMXV2dYcqUKcNsNpvSWvP4448Xd+fA5UghwYsQcaJ1qAXaFE4NTq3p4e6FqWiwMrRXeuuTO8CvYberC0fbF/Jg3QOQAA06CabfD5N+DUYTBkvrKZLx6MkRIpjs7Gznpk2btrZ/ZvcxY8aMuhkzZnS6JyeeJHgRIk6cWgddZM7V86LRuAKZQbkpVDRY6Z/doX62VuKSeXE6YMnDsHw2pYY8HrNczNa0k/loyqyWsQT5mMQuRyyn0+lUBoNBMheiTU6nUwHOUO/LbCMh4sSpg2cYjMpTNtIoBX+6aBRj+md2qlkX/NdTiXC2dcct/iMsewzGXsVtOXN42zGVGmOW3ylh720kjgSbysrKMt1fTEIE5XQ6VVlZWSawKdQ5knkRIk6c7uAkkFIKp9NdVgKG987gw1smd/p+vqWiLllht2IXfPkkjL8WZj6JZc5KwBrW3kYSvByZ7Hb7DQcPHvz3wYMHRyG/PIvQnMAmu91+Q6gTJHgRIk50iMyLt2wU4v1IdflU6a+eBqMZpt0H+O5tFM72ADEfnYiD448/vhSQVQdFp0nkK0ScOLUO+iXtmW0UKjMTKb/gJdaZjbqDsO41GHMFpPfyu2c42wPEY+8lIUT3IcGLEHHiCl6C93s4NWiinHnxKxtF7bLBrXwKnDaY8hvvIcm8CCGiRYIXIeLEqYNnGJQCp9OVeYmmLisbNVTAmhdh9GWQ491c1mdX6dDjCjxXCCGCkeBFiDjR7ZSNCLEOTKS6bG+jVU+DrQmm3BH0/q0bdlWrn1NiFyFEWyR4ESJOQk2VjlnZSAV/HlVNVfD1c64NFfOGBb1/sGxTsIBGCCFCkeBFiDgJ1bCrlGvrgKg37IZ4HlXfPA/WOjj1zlZvhSobBTsmPS9CiLZI8CJEnDidOmiGwagUOiZTpWO8zoulDlY9A4XnQu/Rrd4O1bAbbDzS8yKEaIsEL0LESeh1Xlr2NormV3jMy0ZrXnSVjU69K+jbnq0Qwsu8SPAihAhNghch4iRU2chgAId3b6Po3S+mGzPamlzTowefDv2PD35/7zov4fS8RHd4QogjiwQvQsSJU4Mh6MaMnrJR8LJSpPxnG0Xtsi7fvgoNZSGzLr73DJ55kbKRECJ8ErwIESeOEA253tlGOrqNtTErG9maYcUTkH8K5E8KeVqoqdLQ+ueUhl0hRFtkbyMh4kRrjTHE6rIOZwwadmO1MePqf0NdCcx6rs3T2gpeAqMXmSothGiLZF6E6KRGq50HPthEbbOtQ58Luc5Ll+xtFKWLNtfC8tmuXpdBU8K6fzj3lsyLEKItcQtelFIDlFJfKKW2KqU2K6V+7T6eo5T6TCn1vfsx231cKaWeVErtVEptUEqNj9fYhfC1bk81r3xVzMqdFe2eu7eykX1VjUAbDbtK4V5gN6oZCP/tAaJ03a+ehqZKmH5/u6e2mXkJIJkXIURb4pl5sQN3aK1HACcBv1RKjQR+CyzWWg8FFrtfA5wLDHX/70ZgTtcPWYjWGq0OAEqqm9o9d8qjXzD5r19413EJPvPGUzaK8m2xU7AAACAASURBVFTpaG/M2FAOX/3TtZpu3+PaPd3bsCv5XiFEJ8XtnxGt9QGt9bfu53XAVqAfcCHwivu0V4CL3M8vBF7VLquALKVUny4ethCtNNnCC160z0aLnqdtlY20ju4Xve+1olKWWf53sDXCtPvCu38HMi9CCNGWw+J3IKVUAXAc8DXQS2t9AFwBDtDTfVo/YK/Px/a5jwVe60al1Bql1JqysrJYDlsIAJqsdgAO1DS3ed7B2pb3PTtGt1U2ci1SF6OG3c5et6oYVj8PY69utYdRyPu3sc6LEEJ0RNyDF6VUGvAOcJvWuratU4Mc060OaP2c1nqC1npCXl5etIYpREiestH+djIvu0obvM+dnsxL0HVeYrRIXTQbdhc/CMoI0+4N+yNtrfMihBAdEdfgRSllxhW4vK61ftd9+JCnHOR+LHUf3wcM8Pl4f6Ckq8YqRCie4OVATTvBS1m997kn8xJ6nZfQPTGRMvgFL5247r61sGk+TLoFMlslP9u4v5SNhBDREc/ZRgp4Adiqtf67z1sfAte6n18LfOBz/CfuWUcnATWe8pIQ8dTs7nkprbNgtTtbvd9gsdNotVNU4cq8pCQY2+558SsbRZMK8qyDtIZPfw+peXDKrzv0UU/PjWRehBCdFc9F6k4BrgE2KqW+cx+7F3gEeEsp9TNgD3CZ+70FwHnATqARuL5rhytEcJ7Mi9ZwqLaZATkpfu+P+eOnpCQYOWVwLuBq8HW00/MCrhlH0fyi942TIr7urs9hz0o4/++QmN7B+0vPixAiOuIWvGitVxD6F8DpQc7XwC9jOighIuAJXsA14ygweHE4NXXNdm9ZSWvXwnYQKvPierQ7Y7e3UcTXXfkUpPWG434cwf39H4UQIlJxb9gVortrstq9X8glNU18f6jO+57D2dJTvr+6ZbZRg8UV8ARd58V9MWe0My++zyO57sGN8MMXcOJNYErs8Mel50UIES0SvAjRSU02B/2zXdmW55ft5szHl/Hp5oMAlNa1BCzl9Rb6ZycDrj4YaLtsZHdGeap0Zxt2l/0NzKkwIbKKrQQvQohokeBFiAjYHE6eW7aLZpuDRquDnNQEslLMbDngmu2/p9K1BUBJtf/aL4Pz0gCoa26/bOSM8rbSfmWjjn64eCVsed81wyg5O6L7d2RvIyGEaIvsKi1EBN5ft5+/LNhGbZOdJquDlAQjfTOTqW50bc6YkWSmwWLnzrfX+31ucF4aS3eUtZl5MboP2h0aQ0Jsxt/hAGL5390zjG6L+J6SeRFCRItkXoSIgMno+gLeU9lIoyd4yUryvp+aaGJ1USW7yxv8PndCgStr0eBu2A1WvlE+s41iVTbqUABRug12fgYTb4SElPbPD6HNht1Wy00KIURoErwIEYGsZFdKpLLBSrPNQZLZSN+sZO/7Dq1pcs9CSk0weo97zvGUjYwhVtgFsDudUd3bSEVaNlr1NJiSYMLPOnV/ybwIIaJFghchIqDdqYKKBqs389InsyV4sdmd1LtLQ+/efAoAfTKTSE10BTLhlI2inXnxX2E3zA/Vl8H6eTD2Kkjt0an7yzovQohokZ4XISJgc7iDl3oLzTYHKQkmv7KR3emk2eZabTcvPZEnrzqOiQU53sXpPMFLm2UjraO7t5HvCrvhXnj1v8FhgZM7v8SSrPMihIgWCV6EiIDdHbxUNlhRCpLMRvr5lI2sDu3NvKQkGJk5ti8AVQ1WAOrd67y0NdvI4YjuInV+U6XD+YCtyRW8FJ4DuUM7fX8pGwkhokWCFyEiYHc63Y+uICbRZKBPln/ZqNFqx2RQJJpaqrPJCWGUjfzWeYmeDq/z8uU/oLE8KlkX8Mm8SLFaCNFJ8s+IEBHwZF48TAZFr/REEoyuv1I2h5MGi6sXxjdQMLm/wW3u4CfUxozgWucluivstlys3etu+RCWPAxjroCCKdG5v/dnlcyLEKJzJHgRIgKezIuH0agwGQ0su3saADVNNsrqLKQl+ic3vavnuoOfYAkQz7Fo723kn3lp48QDG+C9m6D/CXDBk1FbVc7gbdiNyuWEEEcxKRsJEQG7s3XmBaBXhmvPn2eW7AJgSM80v/M8+xbZ28i8eGcbOaKbefFfYTfEhbWGj26HpEy44nUwJwU/L6L7R+1SQoijnGRehIhAYNnI6G7kUEphNrZ8S6cmtv79wGhQ3tlKbZWNXAFSFDMvvs9DXXbX57B/DZx6F6T3itq9oSVwE0KIzpLgRYgI2Bz+ZSPfgMXk05GaYjYSyKgUdocn89L62p7AIupTpdtr2G2qhv/7NeQcA+Oujt6Ng9xfCCE6Q4IXISLgcAZmXlq+mX0DmcDeGHB9iXvKTsGCCN9F6qLasNveCrsL7oLaEpj1PJiTg53RKTJFWggRLRK8CBGBUD0vAAk+U6M92wD4MhqUt+wULDjxfMl36d5GWz6EjW/BaXdD/wlRu6cvqRoJIaJFghchIhBYNjL6lIrMxnaCF6XabNj1/ZKP6t5Gvs99XzSUu5p0+4yFKXdE74YBJPMihIgWmW0kRAQCy0a+mReTT9mottnW6rMG34bdIMFJWLOCImAIVTZacCc018C1H4LRHLX7BZI9jYQQ0SKZFyEiYGs128in58UnIjl3VO9WnzWoll6YYF/ofkFGrBt2N70Lm9+Dqb+FXsdG72ZBSNlICBEtknkRIgL2NmYbedIa159SwL3njWj1Wf+elyDBi8+vFFFdpI6AoKi+zJV16XscnHJb1O4TipSNhBDRIpkXISIQ2LDr2/PidL/XLyvZr//Fw6CUt2fG2F7mJSqjdV/LN77SGj7+DVjq4KJ/gTH2v8dI5kUIES0SvAgRgcAp0L49Lw7t3qwxyBov4M68ONufbRTq/Uj5Bi+9ShbB1g9h2r3Qc3j0bhLuAIQQohMkeBEiAq1X2G35YvbENUmm4H+9DEr57G3UXs9L9MtGRhwM2fQ45A6Dk2+N2vXDH4cQQnSO9LwIEYG21nnxzEQKlXkxGMBqD73Crn/PSycH6ntd97X+x/gxqbW7XHsXdUG5KBy6/VOEEMJLMi9CRCCwYdcYpGwUKvNi9Mm8BNvvJ1ZTpZVSpNDMLab3Ket3Bgw/P2rXFkKIriTBixARsAVmXoytG3aTQmZeWhp2u7TnpaGclxP+SppqZu/IG6UHRQjRbR0eOWMhuhmHo42ykadht42eF0ebexu1PI/qOi+f3MFEw3aWOUajc4+L3oWFEKKLSeZFiAjYnU6/rIgxSM9LqMyLUSlv5ibY2icqFmWjH5agtnzA32yX8RPb74KWq4QQoruQ4EWICNgc2i848d0SIJyykT3cslE0/obaLbDgbsjK53mHq88lmr00QgjR1SR4ESICDmdA8OI7VdpdUQpVNjIaWs4JlnnxX7guCkHGl/+A8u1w/mwsJLjv2/nLCiFEvEjwIkQEbA6n32wi3xV2vbON2igbeQTrafE91ukgo7kWVv4Ths+AoWf63KST1xVCiDiS4EWICNjbyLy09LwE/+ul/GYTxXhjxjUvgqUGTr3TfwyHWfSitaz0IoQInwQvQkTA7tR+i9D59rx4F6kzhd4ewCNo2aid98Nma4KvnobBp7s2X/QhZSMhRHcW1+BFKfWiUqpUKbXJ51iOUuozpdT37sds93GllHpSKbVTKbVBKTU+fiMXRzu7w+mXWfENOE46Jgdoo+elnXVcfI91KsZY9xo0lMLk37R6K5rbDgghRFeLd+blZeCcgGO/BRZrrYcCi92vAc4Fhrr/dyMwp4vGKEQrdocmyeRbNmr5q/Tva09g4W1TQk5H9l/+v52p0pEGGQ4brHwS+k+EgslB7hHZZYUQ4nAQdvCilJqslLre/TxPKTWoszfXWi8DKgMOXwi84n7+CnCRz/FXtcsqIEsp1aezYxBHnyXbS1st799RdmfozEtaoonhvTNCfra9FXR9rxVxkLHpHajeA1N+E/QiUjYSQnRnYQUvSqkHgHuA37kPmYHXYjSmXlrrAwDux57u4/2AvT7n7XMfCxzrjUqpNUqpNWVlZTEaouiulmwv5bqXVjNnya5OXaetht32+AYnxqB7G7U8j6ix1umEFY9Dz2Nh6NkhTpLoRQjRfYWbebkYmAk0AGitS4D0WA0qhGD/2raaoqC1fk5rPUFrPSEvL68LhiW6k72VjQAcrG3u1HXsbSxS1x5DB2YbRZQh2b4AyrbB5NtDrnInZSMhRHcWbvBi1a65jBpAKZUauyFxyFMOcj+Wuo/vAwb4nNcfKInhOMQRqMnmACAlIfhMoHDZnU4SfDYhMnVgKdz2ykKGzpSNtIblsyG7AI69OORpnZrFJIQQcRbuv7hvKaWexdVn8j/AIuD5GI3pQ+Ba9/NrgQ98jv/EPevoJKDGU14SIlyNVlfwkhxiAblw2R3aL9vSkQyJ77nBMy9tv9+m3Uuh5Fs45ddgDL3vqoQuQojuLKxdpbXWf1NKnQnUAsOA+7XWn3X25kqp/wBTgVyl1D7gAeARXMHSz4A9wGXu0xcA5wE7gUbg+s7eXxx9PMFLSmLnNlS3OZyYfTIvHZkV1JGyUYejjGV/g7TeMPbqNk+TxIsQojtr919wpZQR+K/W+gyg0wGLL631VSHemh7kXA38Mpr3F0efJmt0ykYOpw7abBsO/0XoWr/vt8JuR6KXH5ZA0XI4+2EwJ7V5qpSNhBDdWbtlI621A2hUSmV2wXiEiClP5iUpxOq34bI5dYeadH3597S0VzYK86Jaw+I/QUY/mPDTiMYlhBDdRbi582Zgo1LqM9wzjgC01r+KyaiEiJFGqz0q13E4dYemR/tqbzZRRHsb7VgI+9fABU+2m3Xp0HWFEOIwFG7w8rH7f0J0a57Mi6OTGwE6tfZb5r8jjO017HZ0byOnAz5/CHIGw7i2e106dF0hhDhMhduw+4pSKgEodB/arrW2xW5YQsSGZ6q0sxPBi9YarSNfur+94KTDexutfQkObYJLXwSjOawxSOwihOjOwgpelFJTcS3VX4Tr39MBSqlr3cv7C9FteBp2nZ1IvHg+G3HDrm9ZKEjXmd9124sy6kth0YMw6DQ4dlbYY4ho5V4hhDhMhLvOy2zgLK31aVrrU4GzgcdjNywhoq+myUZxhatlS3ci8+JwRy+R7g8U1RV2P/092Jvg/NkdSqfEY2+jyUNyAbj4uFa7evC780Z09XCEEN1YuMGLWWu93fNCa70D1/5GQnQbVz63itpmV8OuoxOpF0/JKdSu0e0xtDNV2rcRuM0Mye7lsGGea0G63KEdGkM8ykaDclMpeuR8JhTktHrvxyfls+1PgRvMCyFEcOE27K5RSr0AzHW//hGwNjZDEiI2th6o9T7vXNnIk3mJdJ2XlufBrtHeOjAA2K3w8R2QlQ9T7ohgFIdf2Uj6cIQQ4Qo38/ILYDPwK+DXwBbg57EalBCxMH5glvd5NMpGkc82ansqtApnqvRXT0H5djjvb2BO7vAY4lE2ao/MgBJChCvczIsJ+IfW+u/gXXU3MWajEiIGMpJbKp2dKxu5HiP9rlXt9LyEOterYhcsfRRGXACFZ3V6DIcLCV6EEOEKN/OyGPD99S4Z1+aMQnQbTg3De6d7n0d8HU/mJQrbA7SXvWn1ttbwf78GYyKc+1hE94fDsWh0eGaDhBCHp3AzL0la63rPC611vVIqJUZjEiImnD5L+ndmnRfPZ6MRvISKXQzKFWC1atj99hXX/kUX/AMy+kR0f9f1D79I4XDMBgkhDk/hBi8NSqnxWutvAZRSE4Cm2A1LiOhzOLV3J2hnJ1IvntV5lVJ8/KvJOJ0d+7wncFAq9Be2QSmcWvtnI+oOwaf3Q8EUGH9tJEP3kjhBCNGdhRu83Aa8rZQqATTQF7giZqMSIoqG3LuAa07Ox6E1ZoM7eOlE2ciTtDEqxbF9O75fqScgaSv74XpP+wcZnjVdZjxxREcfkwb3iPcQhBCHuTaDF6XUCcBerfVqpdRw4CZgFrAQ2N0F4xOiUyx2B3an5qUvi5iQn02CyRO8xG+ROk/ZqK3Pq8AAZ/dy2PgWTLkTcodEdmMfka5RE2vL755GbprMBRBCtK29ht1nAav7+cnAvcDTQBXwXAzHJURUHKqxeJ87tMZoUCgVnZ6XiBep85aN2su8uDlssOBOyBwY4ZourR2eoQsMyEkhOcEY72EIIQ5z7ZWNjFrrSvfzK4DntNbvAO8opb6L7dCE6LySmpbWLKdTY1DK208SKU+PS+SL1Lk+l2gM/buDX2lp1Rwo2wZX/gcSotMnfwRXnYQQR4H2Mi9GpZQnwJkOfO7zXrj9MkLEzQHf4EW7ggKjUlFZYbeN2KNNnsDEbGoreHGdlGItgyWPQOE5MPy8yG7YxvWFEKI7ai8A+Q+wVClVjmt20XIApdQQoCbGYxOi00qqmwFINhtxOKNTNnJ0cnsAT7kpoa3Mi/uc43bNAYcVzv5LRPcKRUIXIUR31mbworX+s1JqMdAH+FS3rKluAG6N9eCE6CxP5sXqcOLwLRt1ZoVdZyf3NnJ/zmxqq+cFhqh9DN3/Ppx4I/QYHNG9QpE1VYQQ3Vm7pR+t9aogx3bEZjhCRI/DqflyZ4X3eV2zDaNBYTR0tmzkeuzsInXmNnteFL81/Qe7KYWEU++O6D5tkdhFCNGdRVi1F+Lw9/m2UnaXNzB1WB4AFQ1WDNEoG3VyqrQnYxOybFSxizcdv+EM4zo2HXMDpEZ/3ROJXYQQ3ZkEL+KItX5vNUaD4tLj+wNgsTsxRqNs1NmeF/fHEoM17Drs8J8r6Ukls22Xsr3gx5EOs50xSPgihOi+JHgRR6zdFQ30z072W/QsOmWjTva8tFU22jQfynfwsOlmnnLMAlNsFmyT2EUI0Z1J8CKOWMUVDRT0SCUjyew9ppRn08POZF5cj5H2vBhCBS8OOyz9K/QezTLDia7xRjzKtrXa8FEIIboRCV7EEUlrTVF5I4NyU71bAoBrpo/q5CJ1np6XSLMXLbONAv76bZgHlT/A1HtR7j2YYpUhUfI3XwjRjck/YeKIVF5vpd5iJ79HCiafDInRoFyL1HVwJ2hf2rtIXee2B/Br2HXYYdmj0GccDDsXd+wSsynNkncRQnRnErwcZYorGuI9hC7x4foSAMYPzPYLMgwG1emykaOT67x4F6nzXedl20dQVQSn3gnupmKIYdlIml6EEN2YBC9HkS0ltZz22BI27jsyF0e+e/56Xlixmw/XlzD3qyImDsph7IAsTEafzIu7bOSI5wq7nu0BPJkXpxNWPglZ+TDsPL9rx2pW0GG6qbQQQoRF9ic6ipTWuZbKL6+3tHNm9/TWmn1+r2eNd02RNgaUjQwG6ETs4v1spAGAzeGqWXnLRpvmw/61cOHTYHDtqOyJWWLW8yKFIyFENyaZl6NIk9UBuNY7OZzYHU7umb+BXWX1Ub1ufg/XDsxGnwjAoNw9L1EoG0Xa82J1uD6fYHJHUV/+A3oeC2Ov9hun72O0SdVICNGdSfByFGmyuYIXqyO2wct/Nx9k1Q8VYZ+/s6yeeWv2cssb6zp0n/fX7eevC7fx5c5yrEECsgE5ruDFZGj5z9ygXAFBVNZ5iTB4sbnHajYa4PvP4NAmOOnnEDBOiGHmRYIXIUQ3JmWjo4g3eIlx5uWmuWsBKHrk/LDOt7szEbVNtg7d57Z53wEwZ8ku/mfKoFbv57uDF6PRv2ykFHFdYddTNkpTTfDxbyC3EEZf7ndOrFfAlbKREKI7k8zLUcRTNop18NJRtc2uoKXeYgfg822HvGMN187S1iWnnNQEAL+p0gbvCrud2VXa9WiMMMDw/Pmfvm8O1O6HC58Bc5LfOUrKRkIIEVK3C16UUucopbYrpXYqpX4b7/F0Jy3BS8cCg1irbXIFLQ0WO0XlDfz05TXMXVXUoWsETv0dNyDLe8yvYdezt1EUZhtFGgBYHU76UcbY0vdgwk9hwAmtzol12Uj2NhJCdGfdKnhRShmBp4FzgZHAVUqpkfEdVfcRy56X2mYbjVZ7RJ+tc2de7E7NtoO1AKz6obJD1/D9Kk5PMvH+L0/xvvZr2DW4p0rHcZE6ZanlMfOzgIJTbgt6TswbdmNyVSGE6BrdKngBJgI7tdY/aK2twJvAhXEeU7fhCV5sjk50q4Yw5g+fctJfFkf02brmlqBnc4kreFldVOmd1RMO3+/4hIA9gwwBmRejoSUAiYQn8IkosCj/nlu3X8dEwzYqp/8dsgYEPc2beYlwjO2RxIsQojvrbsFLP2Cvz+t97mNeSqkblVJrlFJrysrKunRwh7tmW2ynStc2R5Z58fS8ACzd4fr/rK7ZztYDtR24Ssu3cULgnkG0fFkbDfiVjbYfrOP+DzZ1qIHX6c28dGB4AHYLvHE5Zm3BdMOn5E2+NuSpnoArdrONJHoRQnRf3S14CfYvrt+3jtb6Oa31BK31hLy8vC4aVvfQ2MUNu+FmN3wzLxv21TCyTwYA3+zuWOnIo9VuzbT8h6O8K+y6Xl//0je8+lUxJTVNYV/f6e156WAA8NU/XRsvznouaJ+LL+/2ABJkCCFEK90teNkH+ObZ+wMlcRpLt9PVs42abeHdp7bJRq+MRO/rn5ycz4CcZL7eHf5aMb7MxtBf+K6NGVsCK0945QkWVv1QweKth9q8vneRuo4EFjX7YdnfYPgMGHx6u6fHumwkhBDdWXcLXlYDQ5VSg5RSCcCVwIdxHlO30dKw2zWzjRrCbOCta7aTkWTm9OE9AbjouH4cPzCbDR3Yg8k3y5NgMrZ63zvzKGC2kdNn5tAPZfVc+dwqfvbKmjbv5akwdahh97P/Be2Es/8S1umxniothBDdWbdapE5rbVdK3QL8FzACL2qtN8d5WN1GcxctUufRYLGTm5bY7nm1zTYyks08ffV4Gqx2ksxGemUkUdFgRWsdVunEd+pzQpDMi+eIa1dp5c2eeD729Q+V3kXv2r2Xs4NTpXcvh03vwGm/hez8sD4S66nSQgjRnXWr4AVAa70AWBDvcXRHXd3z0mBpP8OzuaSGlbsqmDI0l+QEI8kJrqxJVkoCVruTJpuDlIT2/zP17a5ps2FXuVbh92RPPJ8LXOTO6dQhl/93dmSqtMMOn9wDmQNhcvBp0cHEeqq0EEJ0Z92tbCQ6oSv2NvIt34Sz7svLXxYBUNAj1e94dooZgKrG8LYM8A3IgjfstixYZ1CqpefFPdzAGMF3BpTHku2lHKhp8i5SF1ZgsfYlKN0MZz8E5uRwfhT/a0vsIoQQrUjwchRp7oLMi+/aLA1hLPHfYLWTm5bAfeeP8DueleJa2r+qwRrWfT2BGYQIXtxBQOuykbsEFHB+ZZD7XvfSamb+80tv1qbd4KWxEr74MxRMgREzw/o5AscrsYsQQrQmwctRpDHG67yAa5Vc7/0s7Wde6i0O+mWnkGT2b7L1ZF6qw8y8+O6FZApSzvEGL8q9MWNA2Sgw9RIYvHiCnLI6i7fnpd2q0ZKHobkGznmkw80rnsCoE2vpCSHEEUuCl6NIV0yV9s281IcRvDRY7KQltp4dlO3eVLGqMbzMS7NP5iVYnOAtGynXxow6YLaRw+n/Z1Je739f36AsrJ6XQ1tg9QuuvYt6jwrrZ/DluXRn9mASQogjlQQvRwmnU3szLrHsefH9kq9utFHTTuakwWInNUhDbpa356XjZaNgAstG3syL+zFwTZqfv7aWDfuqva/tPlsqOLyzjUIEL1rDwnsgMR2m3RfW+AN5Mi8dWPhXCCGOGhK8HCWafXaStsUwePHNvPx5wVbGPvhpm+fXW+ykJQYJXpI9PS/hlY0aw+ivAc/2ALTqeQkW/Kzf2xK82HwyM56AJ2TmZcv7sHuZK3BJyQlrXIGUN3iR6EUIIQJJ8HKU8P1yj2XZyO7s2LUbLHZSgwQvCSYDaYmmiMpGwXjXeQlYpM4TGjS3E/z4ZV50Gz0vtQfgo9uhz1hXyShCnmt3ZgNJIYQ4UknwcpRoimHw4rupYUd2ggZX5iVY8AKQm5bAwZrmsK7T3k7Z3hV2vVOl3W+4H30zLzeeegwANU0tWR+7T7bK4W3YDRK9fPEQWBvgkhfAGPkyStKwK4QQoUnwcpTwZCaSzcaoBy++fS72doIIXxa7A5tDB23YBRg/MJuFmw/yx//b3OGgKFDLInUKg6Ele+IMUjYa1S+TJLPBL3ix+dxfh2rYPbAB1r0OE2+E3KGdGq/B/TfTIdGLEEK0IsHLUcLz5ZyZbI56w65vX4a9A0GGZwXeUJmXkwb3AOClL4vYU9no915HgxnfXaWDlY18M1PJZiOZyWZqm1pmS/lnXlyPfpkXrWHh7yA5G069s0NjCzpeadgVQoiQJHg5Snh6XjKTzVFf58XuVzZqfW1niG/gBvdU6lDBy+Qhud7nFrt/T0rbvTXB1nkJXjZqmW3UOnjxy7z4BC/OYD0v2z6C4hVw+n2uAKaTWspGEr0IIUQgCV6OEn6ZlygHLw5H25mXUNkYzzowwWYbAfTNSmb2ZWMBsARMZW67PNX6PW/ZKHC2Ea3LRskJhiDBi/86L0r5TJW2W+DT30PeCBh/XRvjCp+s8yKEEKFJ8HKU8MymyXCXjaL5G71vX0awoCJYiWdnaR27ylybIYbKvAD0zkwCWq8K7AmILj2+PzdMHtTuGP1mGxlaykaeoTX6lY1MZCSZAxp2/YMXo2/J6JvnoaoIzv5zp5p0fXnXeemaPTSFEKJb6Xa7SovIeDIL2SlmtHYtVJdoCt4o21G+JZxggYrrff97nfH3Zd7noRp2AZLMrvi6VdnIXcYZ3S8zSCDWkbKR64lf2SjBVTbadrDOe8zm9O958fa7NFXBssdg8HQYMj3kz9FRSjIvQggRkgQvRwlPZiE3PRFwNctGK3jxDViC9aIEBjSBwUZ6kjnktT1jDFwB13NNk1GhWiUQQ3/hG5UKWKSOVtdPMhvISDazv7qJnaX1lFQ38as31/mN3zMbiC/+4tq/cvfEJQAAIABJREFU6Mw/hrxnJGSqtBBChCbBy1HCk1nITfMEL3Zy3PsHdcbmkhrOf3KF93WwZuDANVg8vS6XjO/PqH4ZDO2ZFvL6iabgmRfP1GWTQYVept+Hp4ekZXuAgNlGNt+NHQ3e+1753Fet9jlyOLUruFjzInzzHEy8CXqPbncMHWGM0Qq7BiUzmIQQ3Z8EL0cJz1Tg3DRXwBLOponhmPtVsd/rYM3AgZmXQ7UWAE4tzOXCcf3avL4n8xLYsOtpEjYZDK02YvzRSflBruQ6qaXnxXXUkwXyjPGBC0aSl57IqYV5PLvsh1aBC7h6fHqqGvjsAThmKpzzcJs/QyRunT6E3RUNnDu6T1Svu/K30ymvt0T1mkII0dWkYfco0WhzYDIoslJcwUtDlIKXwMAhWOYlsJRUWudaNTfPXcJqS6K356XlGo8u3Ma76/YBrrKR72JxL19/AtOG9Qw5Ts9sIx3QsAvQJzOJ609xNf+eMiSXhy4Kvhu01vAb9RrYmuC82WCITvnNV//sFN666WQyk0OX1CLROzOJUf0yo3pNIYToapJ5OUo0WR0kJxi9zbHRyrwENscG22MoMPNS6s689MpIavfqSZ7Mi0/Z6Jklu7zPTQb/+DvUZomBexsFW7nWZHSfZWsG7WBQbioAaTQywbCD0w3ryFQNGFc7mWFcBafcAblD2v0ZhBBCRJcEL0eJZpuDZLPROy3Zs7ptZ4WXeQkIXtyZl54dyLwENux6GA3KbwxB9xvCN/Piau81Oq1gqaO/KmWc2kWZzuIUxyGY+xQUrQCHlRN6juGjhFqGqv0kKhtNOoEDOoc01cyHzlOYOfXedscvhBAi+iR4ibPqRitJZiNJ5uiXHnw12VyZl9QET/Din3n519JdPLt0F+vuP6tD1w1MdATreTn3ieW8cN0EpgzNA1w9L8lmY8jF6XwlGIM37HqYjf4DCBm8uHMvqaVruWvjT/kD1fAwrPCNnyxA9VDX3kSmRMz71lBtMDLXPoLPncexxZlPNene02dGaU0XIYQQHSP/+sbZuAc/4/j8bN75xaSY3qfR6vALGALLRo98sg1wrZ9iMobfCqUCykaezEuiyeB9bnU4ufPt9Xx97xkA7CytpyA3NbxZQgZFgrHlWoElKJPR4Df1OmTZSEGBOkD+pw9Ra0zlees5/PLMEfx54S7WOgvJVnUk9TyG52690udng9GNNpz7qnnoxW/aHasQQoiuIQ27h4G1xVUxu/azS3exuaSGZpuDJJ+yUaM1eM9LRzdtbF02cmVIAjNJ1Y02tNb87b/bWbqjjBF90glXosngnW0UmIExGZRf30vQ2EVrxjs38Yr5r4DirWFP8C99EU0n/JKXHeewUR/DMudYyhIGtPpoZoqZ1DYW0RNCCNH1JPMSR7HedG9naR0Pf7KNIWvTSEkwkp2SQILJQILRQL1Pz0ttc8sy+Babk5QOLP8SGCt4gowks4GaJp/jdif7qpr45xc7ARjZJyPseySajTS7g5bAKdMmn2nP4MrU+NEaFtzJ07Z/06ASOXDeW1Tv741T7261lYHZEDyWj9ZifkIIIaJDMi9xFM7uznO/KuLxz3Z4X+8sraMijHU6DtU2c/Pr3wKuzEVVo5XsFNe029REo1/Py3afZfA7uuN0YOnHk7kJ9oVfWtcy7qG9Isu8NAdmXozKr+/Fr+fF2gAL7oTV/2ae8QJOtvwTW58JroXanNpvp2gAsyl4ycmzYJ0QQojDg2Re4sh3M8BQ/veDzQDcfmYhDqf27gm04p5p9M9OCfm555f9wI5Dro0Pm20OqhpsZLtX1E1NNPkFL77BUGd3nPYEGcG+8MvcwctxA7M4cVBO2NdMNBu85aLAWUcmg8FvFVrvhonfL4KPb4fqPXDSzTz13RnU0ozR4FoXxql1q5/VHKLXRzIvQghxeJFfKWPM6dTeL+1AofpOPALLSnsqG73Pv/6hss3PflNUyfiBWdwweRC7yxuot9jJdteD0hJNfg271Y0+ZaMQs3pCCZzd48mMeKY4+ypzB0nP/vj4Ds2uSjQZvRmhwPEZA3peTJZqePdGeP0SMCXBdQvgnIdRPuUkpVylpsAp3IFrxnjvH+RnEUIIET/yr3IIpXXN3Pfexg5/mQeas3QXJ/x5EfuqGlu919RO5qXMJyNiczjZcailvHPAt6EkQIPFzuaSWiYNziW/R4q3J8Qv8+ITOFU3+QYv0cq8tA5OisobUIoO76mUZG6ZbRSYeTErTc7Od/md6XUeMT3H0LenwaZ34LTfws9XQMEpQMusKK1bmnoD/79NkLKREEJ0C1I2CuGhj7by4foSJg/J5dzRfZizZBdf7izntRtO7NB1Pt9WCkBJdXOrMk97ZaNdpQ3e51WNVna4e1MSTAYO1DSH/NyH60twODUnD+7hN3vI0/OSlWzmYG3L5/0zLx0LXgI3DmzpeWn9hb9hXzU9UhM7NBXbcy3Pyr2eR4WTiw0rKHjnIRIrtnCTCSzaRFPfs0g/4x7oO87vGp4EkaYlWxQYPIbMvEjZSAghDisSvITg2Y/Hs4z8Xxe61kGx2B0d+jLz/JYfbHfgUMGLxe7A6YQfyuu9x6obbXxfWk//7GTSk8wcbCN4eWLRDibkZzNpcA9KfM7LcZeNeqQlsLmk1ufaLZsPdjTTFNj0arEFnyoNsLqoihEdmGWE1rBvDZc1/Id0y0H4dAF9q5r5t3k1Jxq2ka6asDKC0ulPMP3jNAw4eeec80jv2boZ2BOwaK29a8H4ZpwgdM9LgmRehBDisCLBSwi+ZQbfxs49FY0U5KZid2iSE9oOYp7+Yieri1xruATb8yfYMYDz/rGcXWUN/HLaYO+xqgYrh2qb6ZuVTHqiKWTmpcFi51CthetPGYRSir6ZLfsHZXmDl0QqGixorVFKdSrzEjjd2PP5pIA+kfQkE3XNdu+u1u3avQwWPwj7VnMJUKWyYNVi+mHArnL40DmJ1Y5h/ObKe8FgpO7jL4DQK+wOzElhd3kDCaaWXah9gzYIXTYKtfCdEEKI+JDgJRT399Wt/1lHQY+Wcs/u8gYe/GgLy78vp+iR89u8xGP/3e59HmwjxFCZl11lrnKRb4BS1WilssHK4Lw0eqQl8O2e4AvbVdS7vpBz01zr3vtOZfb0mvRITcDm0NQ228lMNlPdZPU28XZ0tlGrzIs7c5MQkMX46yVjWLT1EGeN7NX2BevL4LP7Yf0bkDmA/2/vzqOjLs8Fjn+fTGYme0KAJEDCjmwaWQWKVQQVxbZqxb2ntlptq721V61Frbd4em1pry09Vk/VurbuLZVacVcQLQUURFbRsO8hZCOErPPeP37vTGYmMyEImQWfzzlzZub9/X4zD29C5pl3ZcZ9/PSzYazY28q7t5zBK6v3cPPzq3jhhonU7aimpEdWSOtStOTl/itGs2RTBcXdMgLnVB1ykrb8TA+Vh5qidhsppZRKLJq8RBH8Ebj1QNtg2y0Vh3j/8wrAWao+2rfylrAP9bqG4Nk9TZw7dzHnjuz4g3xfbQO9c9PYXdNAVX0zlYeaGD/AQ6/cNKrqmwOr5gbzD/INbuEoLc5l9c4a8uyYl+722IG6Rid5qW+mINtLXWPLUbe8NIfN2Glq8TmbH4YlEcOKsplxSq/oL+RrhRVPOK0tTYfg9FvgzNvBnU5O+Xp2frqN3bWNgQHBvfPSmTCwO+AsVOcX7eeRm+HmfPv+/unUVbblJZC8uLSFRSmlkoF+1Ywi0r47HlcKWw+0DaKtPNTU7hy/mrDxFAeDkpelmyspP9jI00u3B8rCWzAA9lQ3MMyOETlQ10hVfRP5GZ7A/kSRZitVBJKXth0H/3rdBF64YWIg0eme6Rw7YOOvOdxMQY5T1hilKyua8CSt0SYv4aKNJwFgx3L481mw4FYoKoUfLoGzfwHudAC+O7k/AI8s3hxx+4Hg92u3wm4E/h9tVX0TqSltuzMV5qRFvUYppVTiiEvyIiKXisg6EfGJyLiwY3eISJmIbBSR6UHl59myMhGZFeuY81wN/LDbh7Ts+oQ0GulBDftqI487afUZloatw3IwpNuo/eDdSONftlfW0797JmnuFLZU1OMzTiuBxw4YjrQPUaTkJTfdHWilwBiK2E9/2UNNZQUYQ3V9E0PTaymVTfgaD7Z7zY607zbyhbSE+LUb9NpYB+v/Cc9cCo+dAwf3wSWPwTX/gp5DQ04t7pbBmL55/LusgmeX7wBCx9S4j7S3URh/fBUHm8hNd7Ozypl23r975pEvBs4bWcSa2Ue3+7ZSSqnjJ17dRmuBbwIPBxeKyAjgCmAk0Bt4W0ROsocfBM4BdgIfisjLxpj1XRVgt7oy5nt+zZ3N17HdFPCYZy5j69bhOyjc4/WQRhN1806BSdfAqG+Bu+1b+69e3cBjH2wJeb3gbqNIXTMNzT6yw774t/gMRbleSrplsGqHM8YlP9MTWFwtfJ8fcD6Qoa1rKMS+9TDve5xUvo5FXuBlMAs8LEtxkbPpMHiBt+6GjwZA4UgoPNm571UKef3a78IINIcN2D3c1Epqitj0zJCKXbSu6nPYUQb7N8KulbB5EbQ2QkZ3mHo3TPg+eKNvGTCsKIcnl2wNPA+e8eUK6u5xdWKn6p62osv215Gb7g60QA3o0bnkJSc9lew0d6fOVUopdfzFJXkxxmyAiF0zFwLPG2MagS0iUgacZo+VGWM22+uet+d2WfIyruYNRqVs4lXvnTZoeK/P9WzYtpeeUkWFyeXSls1OV8drsyCvBLr1h279SV96iJtchmypJ40mekklrTvGQMPPIS2XqgjdTf6Wl/BVdQtz0hjUM4vX1+0FoCD1EO4dS7ggZRNNTROB0LVjKuoayctwt3XT1O6B3Sth7xr4z4PgzqB1+m+47V9buXiolz7uOhav38GUSZOY8+9qrj2piQkZu2HfWvh0AYFWorRcp0unqNQmNiOh5zBafD5SaWGYbKdYKqiu68aEjF3M3PIeP/OW0w2nJcf7hD95E+g+CMZfB0NnQN9J4Dryr+HQotDEJng/o9Sw1XOPpCjXaZUqK69jVEleoLxvfvTtFoK5dGCvUkrFVaIN2O0DLA16vtOWAewIK4+4WpyI3ADcANC3b98vHEidL7Tl4i33WewcfiNzNm0IlP263LD8qjRcm99h3/aN9K/Zj3vHSm5LrQ65ttzkUVD+Efz+OehVyhBTys2ucsrJ4/SUNWTRQNbCd6DvSBrzBjFStpBJAwKM2f4xJx9YwjXuKrKknpPnbUMwjPeA79GHnUSgx0lQdApk96Lv3n1c694D8/4GO5dD1da2QAZNg6//AVdeXxa//RbpOUV4XCm8IDu4avo5vPH+6wwvHsKEs21jV1M9lG+AvZ/AntWwdzV89Bi02O4ySWGu9KAgrTy08ppgl3cQn/gGscfk04ibay/5Ou6iEU6sdizL0RgY1ioSnKS4OjFgN1jw2Jbc9LYWlCNNffdz68BepZSKqy5LXkTkbaAowqG7jDH/jHZZhDJD5LE57QeOAMaYR4BHAMaNGxfxnM7Ibj4AwNMt0/hjy8XkZhdza7uNEIVHtvdm/tppVNSdHihNx/lwT8HgwkctGXyrXzWzi5bQsG0lk6v/xGT7mVlr0tluCsnd8CysOUwasMAb9BYroTZrIAfETa3J5ODEW/k8+zQeWPAhc8ZUU9i0A3Z/DOvnA3C9/7otRVA8DsZfDyUTnCQno20zxMKcNPbWNFB5qInS4ly8qS7cLgmdKu3JgOKxzs3P1wqVm2HfOti3js+XLeHlQ2M5YHJZ7Culp1QzuF8Jlbmn8I9VuwOXXX/qjM4NSIlidN9uXD2hL88s297uWMhso060vBRkpyHirOGTm+7mj1eO7tRO3X46sFcppeKry5IXY8zZX+CynUBJ0PNiwP8JGK28S+S0VrLBV8LPW64DIDclhSL7oVWQ7WXGKb14cslW5q/aRUVdaDfQ8L5FrNwe2vqyxjeAu1pLeWHvDib0SmH1nnp6Sg1VJpuDZPC3705gfH4Duzet5ld/X8JBMjDAI7dfx876dC67/33yMz0sO2cajVsrWehrYcuYiRT6B+I21NJ66AAXzX2Tc8acxI8vnhJxjIpfr9w09tQ0sKfmcGAKsceVcuSp0iku6DHEuY28iDnrPmBNTU3bcQM984shbAp1Z2YBdcSTmsK9F58SMXkJboWRTvToeFJT6JbhTI/OTXfz9VN7H/GalXefQ0urj0Wf7efi0X2OeL5SSqmuk2jdRi8Dz4rI73EG7A4BluO0yAwRkQHALpxBvVd1ZSD5vkr2m7bxEIIwqCCLbG8qv7zoZKaPLKK+qYUXP9oJONN5n/j3Vh7/zjgmDuzOGb9dFJj5A3CwoZlP9zpL8n9SAQ2ksd2kUZSTxsHaBnbXNMKAYioKsnjFfvDnZ3pIyytkRB68/KPJnFSYjduVEtg3KDjR2N3g5ozfbaDFV8y3+w7uMHEBKMxN473P9tPiM4FF+Lxu1zFvDwCQn+WhvLbzLRlH49nvTWBbZftNLv060/ICbdPcJw/u0anz/Qv8XTau5AhnKqWU6mrxmip9sYjsBCYBC0TkDQBjzDrgRZyBuK8DNxljWo0xLcCPgDeADcCL9twuYYwh31Szn7bk5axhBWR5U1lzz3Smj3R6w8b1d7phzh1RyN0XjODtW85g6rBCMjypvHPLmdzzjZGB66sONQVaCIJ3Rh7eyxmIevPzq7jvjY0hs5KCuydKi/MCa5t4XHaqdFDy8travYFZSOGDWyMpykkLnO+fIuxNTTnmFXahbQ+lrvCVwT248rToY5mirbAbboRdP+fs4QXHJS6llFKxE6/ZRi8BL0U5di9wb4TyV4FXuzg0AJpaWulBdaDl5f3bz6J3XvtBppeMKWZQz0xGl3QjJUUYHLQhYG6Gm5L8tmuq6pspyW8/BGdc/3wWbtwPwAMLy3hgYVngWK/cyGMrvHaNk/9dsJ4Mj4vJg3uwclvbdgFDImxMGK4o6LX728GwntROdBuFafG1/zf5WyniobMTgZ67fiKNra1HvcO1Ukqp+Eu0bqOEUF9zgG7SQunwk1h9ybnkRFnTw5UijO2XH/EYtG8F2F19uN05o0ryWHbnNLK8qZw7dzG7gs7pEyFhgrZ9g7YdqOfqR5cBkJOWykWjejP7GyM7NWumOOi1/VOEvakpNDb7WFJWwZvr9zE7qOUomuYIyU48k5fOdhvlZrgBXatFKaWSkX7tjMDj8bB02CwKS6dHTVw6Y+LA7pw9vICfTndWjA0e2HvZuGIARvbOoTAnjUxvKot+OoWRvZ3ujMe/M47/mjo4cnzhq9UCtQ0tjOidE9g5+khOG5DPfZeeylPXnhbojmppNby+bi9XPbqMJ5dspb6p/WaS4cL3NgJn1+p46Wy3kVJKqeSlLS8RZOZ0Y+IVdxzz66S5XTx6zXhWbKtsd+zOGcP57cxTQ8rcrhRe+P4klm85wNRh0Tdt9EZIXgBK2k3lji7VlcLMscUhZT2zvWyuaNu7aXd1A4MLsthScYjUFKEkwiJuwXsb3ThlEG5XSiABi4djndWklFIq8WnyEgM9wloisr2pUVtIsrypHSYuELnlBYiYXByN+y49lWVbKvn5/DU0NPvYU3OYwQVZnHXfIgC2zrmg3TXNrYarJvTlJ9OGUKDrnyillIoB7TaKgeDk5R83foXVx7ipX3Dycv+VowOPj6blJZKS/Axmji3mzZ+cCcCemoaQlhVjDDWHm0PG5TS3+sj2pmriopRSKmY0eYmBTG9bA9fokrxO7b/TEU/QDJnstLbXzkk/Pg1phXbvnz3VDWwPWlOloq6J7z6xnMlz3mXeCmd9m+ZWX9s+SkoppVQMaLdRjFw0qjdDi3KOOXGB0BVlMz2pEcuPhTfVRU5aKn9dui0wLRvgzP9bSH2Ts4jdgwvL+PP7m/EZSNW9fpRSSsWQJi8x8ocrRh/5pC8gw+NiyaypEReLOxZ9u2ewdlctc177NFDmT1yAkIG9tYePPCtJKaWUOl40eUlymd7UiAvoHau5l41ib20D2w7UYwCfz/CLl51FjU8f3IMPyirome1l/8FGMiKsK3PhqN689PGu4x6XUkoppclLksvsxIJ0X8SQwmyGFGbz1SFtZReO6s32ynrmrdjJB2UVfK20F5ePL6Fffma766cMLWDrnAuY+rtFDCnI6pIYlVJKfTlp8pLkMryx+xHmZXjIy/AEWlR6ZnsZVtTxmi7v3jolBpEppZT6MtFpIkkuw901LS8dGWj3QhrcU1tUlFJKxZ62vCS5eKwoe/WEfgwqyGLSwO4xf2+llFJKkxd11FJShK8M6hHvMJRSSn1JabeRUkoppZKKJi9KKaWUSirabZSkXrv5qxxubj3yiUoppdQJRpOXJDW8V8dTlL9snr1+AntrGuIdhlJKqRjQ5EWdEHQAsVJKfXnomBellFJKJRVNXpRSSimVVDR5UUoppVRS0eRFKaWUUklFkxellFJKJRVNXpRSSimVVDR5UUoppVRS0eRFKaWUUklFjDHxjqHLiMh+YNsXvLwHUHEcwzkeEjEmSMy4NKbOS8S4EjEmSMy4uiKmfsaYnsf5NZU6bk7o5OVYiMhHxphx8Y4jWCLGBIkZl8bUeYkYVyLGBIkZVyLGpFRX024jpZRSSiUVTV6UUkoplVQ0eYnukXgHEEEixgSJGZfG1HmJGFcixgSJGVcixqRUl9IxL0oppZRKKtryopRSSqmkosmLUkoppZKKJi9hROQ8EdkoImUiMivOsWwVkTUiskpEPrJl+SLyloh8bu+7dXEMj4tIuYisDSqLGIM47rd1t1pExsQ4rtkissvW1yoRmRF07A4b10YRmd5FMZWIyEIR2SAi60TkZlset/rqIKZ411WaiCwXkU9sXPfY8gEisszW1Qsi4rHlXvu8zB7vH8OYnhSRLUF1NcqWx/L33SUiH4vIK/Z53OpJqYRgjNGbvQEuYBMwEPAAnwAj4hjPVqBHWNlvgVn28SzgN10cwxnAGGDtkWIAZgCvAQJMBJbFOK7ZwG0Rzh1hf5ZeYID9Gbu6IKZewBj7OBv4zL533Oqrg5jiXVcCZNnHbmCZrYMXgSts+UPAD+3jG4GH7OMrgBdiGNOTwMwI58fy9/0W4FngFfs8bvWkN70lwk1bXkKdBpQZYzYbY5qA54EL4xxTuAuBp+zjp4CLuvLNjDGLgcpOxnAh8BfjWArkiUivGMYVzYXA88aYRmPMFqAM52d9vGPaY4xZaR8fBDYAfYhjfXUQUzSxqitjjKmzT932ZoCpwN9teXhd+evw78A0EZEYxRRNTH7fRaQYuAB41D4X4lhPSiUCTV5C9QF2BD3fScd/6LuaAd4UkRUicoMtKzTG7AHngwkoiENc0WJIhPr7kW3CfzyoSy3mcdnm+tE4394Tor7CYoI415XtClkFlANv4bTyVBtjWiK8dyAue7wG6N7VMRlj/HV1r62ruSLiDY8pQrzH0x+A2wGffd6dONeTUvGmyUuoSN9Q4jmXfLIxZgxwPnCTiJwRx1g6I9719ydgEDAK2AP8zpbHNC4RyQLmAT8xxtR2dGqEsi6JK0JMca8rY0yrMWYUUIzTujO8g/eOSVzhMYnIycAdwDBgPJAP/CxWMYnI14ByY8yK4OIO3jfe/weViglNXkLtBEqCnhcDu+MUC8aY3fa+HHgJ5w/8Pn/TtL0vj0No0WKIa/0ZY/bZDx8f8GfaujtiFpeIuHGShGeMMf+wxXGtr0gxJUJd+RljqoFFOONG8kQkNcJ7B+Kyx3PpfLfhscR0nu16M8aYRuAJYltXk4FviMhWnG7sqTgtMQlRT0rFiyYvoT4EhtiR/B6cAW8vxyMQEckUkWz/Y+BcYK2N5xp72jXAP+MQXrQYXga+bWdhTARq/N0lsRA23uBinPryx3WFnYkxABgCLO+C9xfgMWCDMeb3QYfiVl/RYkqAuuopInn2cTpwNs54nIXATHtaeF3563Am8K4x5ni3ckSK6dOgxFNwxpYE11WX/vyMMXcYY4qNMf1x/h69a4y5mjjWk1IJId4jhhPthjOD4DOc/ve74hjHQJxZH58A6/yx4PRfvwN8bu/zuziO53C6FZpxvtVdFy0GnCbrB23drQHGxTiuv9r3XY3zR7xX0Pl32bg2Aud3UUyn4zTRrwZW2duMeNZXBzHFu65KgY/t+68F/ifo9345zkDhvwFeW55mn5fZ4wNjGNO7tq7WAk/TNiMpZr/v9v2m0DbbKG71pDe9JcJNtwdQSimlVFLRbiOllFJKJRVNXpRSSimVVDR5UUoppVRS0eRFKaWUUklFkxellFJKJRVNXpQKIiKtQbsHr5Ij7CwuIj8QkW8fh/fdKiI9jvV17Gtlishb9vEHQYuZKaXUCUH/qCkV6rBxlofvFGPMQ10ZzBc0CVhq9ys6ZNr2wFFKqROCtrwo1Qm2ZeQ3IrLc3gbb8tkicpt9/GMRWW838HveluWLyHxbtlRESm15dxF5U0Q+FpGHCdqTRkS+Zd9jlYg8bDcLdInIkyKyVkTWiMh/R4hxkN1U8GngKmAFcKp9nXhs4KmUUl1CkxelQqWHdRtdHnSs1hhzGvAAzv4y4WYBo40xpcAPbNk9wMe27E7gL7b8F8AHxpjROCvc9gUQkeHA5Tibco4CWoGrcTZQ7GOMOdkYcwrOHjshjDGb7DUrcPbf+QtwnTFmlHH2x1JKqROCdhspFaqjbqPngu7nRji+GnhGROYD823Z6cAlAMaYd22LSy5wBvBNW75ARKrs+dOAscCHzlY6pONs5PgvYKCI/BFYALzZwb+hwBhzQEROwdl0USmlTija8qJU55koj/0uwNnrZiywwg6UlQjnmbD7YAI8ZVtLRhljhhpjZhtjqoBTcXY6vgl4tN2FIg+JyFqczUVXAecBCyJ1MSmlVDLT5EWpzrs86P4/wQdEJAUoMcYsBG4H8oAsYDFOtw8wx6vfAAAA8ElEQVQiMgWoMMbUhpWfD3SzL/UOMNM/RsWOmelnZyKlGGPmAXcDY8KDM8b8AKeb6pc4ux8vsAlQpFYipZRKWtptpFSodNtq4fe6McY/XdorIstwkv4rw65zAU/bLiEB5hpjqkVkNvCEiKwG6oFr7Pn3AM+JyErgPWA7gDFmvYj8HHjTJkTNOC0th+3r+L9w3BEl/jNxxrp81b6uUkqdcHRXaaU6QUS2AuOMMRXxjkUppb7stNtIKaWUUklFW16UUkoplVS05UUppZRSSUWTF6WUUkolFU1elFJKKZVUNHlRSimlVFLR5EUppZRSSeX/AaKmYrqQr9yYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play(env, agent, num_episodes):\n",
    "    \n",
    "    state = env.reset()\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    \n",
    "    for i_episode in range(num_episodes + 1):\n",
    "        \n",
    "        state = env.reset()\n",
    "        score = 0                    \n",
    "        time_start = time.time()\n",
    "        \n",
    "        while True:\n",
    "            \n",
    "            action = agent.select_action(state, eval=True)\n",
    "            env.render()\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            score += reward \n",
    "            state = next_state\n",
    "    \n",
    "            if done:\n",
    "                break\n",
    "                \n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)    \n",
    "        \n",
    "        print('Episode {}\\tAverage Score: {:.2f},\\tScore: {:.2f} \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, np.mean(scores_deque), score, s//3600, s%3600//60, s%60)) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0\tAverage Score: 306.86,\tScore: 306.86 \tTime: 00:00:20\n",
      "Episode 1\tAverage Score: 307.91,\tScore: 308.96 \tTime: 00:00:16\n",
      "Episode 2\tAverage Score: 308.11,\tScore: 308.51 \tTime: 00:00:16\n",
      "Episode 3\tAverage Score: 308.30,\tScore: 308.85 \tTime: 00:00:16\n",
      "Episode 4\tAverage Score: 307.87,\tScore: 306.15 \tTime: 00:00:17\n",
      "Episode 5\tAverage Score: 307.78,\tScore: 307.35 \tTime: 00:00:15\n"
     ]
    }
   ],
   "source": [
    "play(env=env, agent=agent, num_episodes=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0\tAverage Score: 310.14,\tScore: 310.14 \tTime: 00:00:20\n",
      "Episode 1\tAverage Score: 307.22,\tScore: 304.31 \tTime: 00:00:15\n",
      "Episode 2\tAverage Score: 306.21,\tScore: 304.20 \tTime: 00:00:15\n",
      "Episode 3\tAverage Score: 306.33,\tScore: 306.67 \tTime: 00:00:15\n",
      "Episode 4\tAverage Score: 306.38,\tScore: 306.60 \tTime: 00:00:15\n",
      "Episode 5\tAverage Score: 306.49,\tScore: 307.04 \tTime: 00:00:15\n"
     ]
    }
   ],
   "source": [
    "play(env=env, agent=agent, num_episodes=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kernel-ml",
   "language": "python",
   "name": "kernel-ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
